
# verify
import sys
print(sys.executable, sys.version)

/Users/nipunshah/anaconda3/bin/python 3.11.4 (main, Jul  5 2023, 08:54:11) [Clang 14.0.6 ]


# Importing the libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tabulate as tb
from statsmodels.graphics.mosaicplot import mosaic

from scipy.stats import chi2_contingency, zscore
from scipy.stats import randint, uniform

from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score

from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV, StratifiedKFold, RandomizedSearchCV

from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import BaggingClassifier, RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier

from imblearn.over_sampling import SMOTENC
from imblearn.under_sampling import RandomUnderSampler
from xgboost import XGBClassifier

# Suppress warnings
import warnings
warnings.filterwarnings('ignore') # Ignores all warnings (optional)


# other imports
from imblearn.pipeline import Pipeline as ImbPipeline


# Global options and themes

# Set pandas display options for better readability
pd.set_option('display.max_columns', None)  # Show all columns
pd.set_option('display.max_rows', 100)      # Show 100 rows by default

# Seaborn theme for consistent plotting style
sns.set_theme(style="whitegrid", palette="muted", context="notebook")  # You can change it to darkgrid, ticks, etc.
plt.rcParams["figure.figsize"] = (15, 6)  # Set default figure size for plots
plt.rcParams["font.size"] = 14            # Set font size for readability

# restrict float display to 2 decimal places
pd.options.display.float_format = '{:.2f}'.format


# Helpers

def tb_describe(df_col):
    """
    Helper function to display descriptive statistics in a nicely formatted table

    Parameters:
    df_col : pandas Series or DataFrame column
        The column to generate descriptive statistics for

    Returns:
    None - prints formatted table
    """
    stats = df_col.describe().to_frame().T
    print(tb.tabulate(stats, headers='keys', tablefmt='simple', floatfmt='.2f'))


# Primitive Utils
def snake_to_pascal(snake_str, join_with=" "):
    """Convert snake_case to PascalCase (eg my_name -> MyName)
    Args:
        snake_str (str): string to convert
        join_with (str): character to join the components, default is space
    """
    components = snake_str.split("_")
    return join_with.join(x.title() for x in components)


def format_pct(val):
    """Format a val as percentage i.e max 2 decimal value & adding % at the end"""
    return f"{val:.1f}%"

def to_percentage(value):
    """value is expected to be a normalized float value in [0, 1]"""
    return format_pct(value * 100)

def calc_iqr(series: pd.Series):
    """
    series: array of numerical values
    """
    Q1 = series.quantile(0.25)
    Q3 = series.quantile(0.75)
    IQR = Q3 - Q1
    return Q1, Q3, IQR


def draw_countplot(
    df,
    colName: str,
    *,
    label=None,
    rot=0,
    order=None,
    sort=True,
    palette=None,
    showgrid=None,
):
    """
    Draw a count plot with value labels and optional features
    """
    # prep (meta) --
    xlabel = label if label else snake_to_pascal(colName)
    priority = None
    if order is not None:
        priority = order
    elif sort:
        # sort by count
        priority = df[colName].value_counts().index

    # plot (crux) --
    ax = sns.countplot(data=df, x=colName, order=priority, palette=palette)
    # display count above each bar
    ax.bar_label(ax.containers[0])

    # Calculate & mark percentages
    total = len(df[colName])
    for p in ax.patches:
        freq = p.get_height()
        percentage = to_percentage(freq / total)
        ax.annotate(
            percentage,
            (p.get_x() + p.get_width() / 2.0, freq / 2.0),
            ha="center",
            va="center",
        )

    # aesthetics --
    plt.title(f"Frequency of {xlabel}")
    plt.xlabel(xlabel)
    plt.ylabel("count")
    plt.xticks(rotation=rot)
    if showgrid:
        plt.grid(True)
    plt.show()


# list all files in current directory
!ls

EasyVisa.csv            notebook_easyvisa.ipynb


# Load the dataset
df = pd.read_csv('EasyVisa.csv')
# backup of original df
df_original = df.copy()


# Peek the dataset
df.head()


# Shape (Size)
df.shape

(25480, 12)


# Data Types
df.dtypes

case_id                   object
continent                 object
education_of_employee     object
has_job_experience        object
requires_job_training     object
no_of_employees            int64
yr_of_estab                int64
region_of_employment      object
prevailing_wage          float64
unit_of_wage              object
full_time_position        object
case_status               object
dtype: object


df.nunique()

case_id                  25480
continent                    6
education_of_employee        4
has_job_experience           2
requires_job_training        2
no_of_employees           7105
yr_of_estab                199
region_of_employment         5
prevailing_wage          25454
unit_of_wage                 4
full_time_position           2
case_status                  2
dtype: int64


# Columns Information
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 25480 entries, 0 to 25479
Data columns (total 12 columns):
 #   Column                 Non-Null Count  Dtype  
---  ------                 --------------  -----  
 0   case_id                25480 non-null  object 
 1   continent              25480 non-null  object 
 2   education_of_employee  25480 non-null  object 
 3   has_job_experience     25480 non-null  object 
 4   requires_job_training  25480 non-null  object 
 5   no_of_employees        25480 non-null  int64  
 6   yr_of_estab            25480 non-null  int64  
 7   region_of_employment   25480 non-null  object 
 8   prevailing_wage        25480 non-null  float64
 9   unit_of_wage           25480 non-null  object 
 10  full_time_position     25480 non-null  object 
 11  case_status            25480 non-null  object 
dtypes: float64(1), int64(2), object(9)
memory usage: 2.3+ MB


# Missing values
missing_values = df.isnull().sum().sum()
missing_values

0


duplicates = df.duplicated().sum()
print(f"Number of duplicate rows: {duplicates}")

Number of duplicate rows: 0


# Statistical summary (only numerical columns)
stats = df.describe(include=['int64', 'float64']).T
stats


# total application with -ve no_of_employees count
df[df['no_of_employees'] < 0].shape[0]

33


df['case_status'].value_counts()

Certified    17018
Denied        8462
Name: case_status, dtype: int64


df['continent'].describe()

count     25480
unique        6
top        Asia
freq      16861
Name: continent, dtype: object


df['continent'].value_counts()

Asia             16861
Europe            3732
North America     3292
South America      852
Africa             551
Oceania            192
Name: continent, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, 'continent', label='Continent')


df['education_of_employee'].describe()

count          25480
unique             4
top       Bachelor's
freq           10234
Name: education_of_employee, dtype: object


df['education_of_employee'].value_counts()

Bachelor's     10234
Master's        9634
High School     3420
Doctorate       2192
Name: education_of_employee, dtype: int64


education_counts = df['education_of_employee'].value_counts()
education_counts.plot.pie(autopct='%1.1f%%', figsize=(5, 5))
plt.ylabel('')  # Remove y-label
plt.title('Education of Employee Distribution')
plt.show()


df['has_job_experience'].describe()

count     25480
unique        2
top           Y
freq      14802
Name: has_job_experience, dtype: object


df['has_job_experience'].value_counts()

Y    14802
N    10678
Name: has_job_experience, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, 'has_job_experience', label='Has Job Experience')


df['requires_job_training'].describe()

count     25480
unique        2
top           N
freq      22525
Name: requires_job_training, dtype: object


df['requires_job_training'].value_counts()

N    22525
Y     2955
Name: requires_job_training, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, 'requires_job_training', label='Requires Job Training')


tb_describe(df['no_of_employees'])

                    count     mean       std     min      25%      50%      75%        max
---------------  --------  -------  --------  ------  -------  -------  -------  ---------
no_of_employees  25480.00  5667.04  22877.93  -26.00  1022.00  2109.00  3504.00  602069.00


print('Skewness : ', df['no_of_employees'].skew())
print('Kurtosis : ', df['no_of_employees'].kurt())

Skewness :  12.265260077708275
Kurtosis :  206.33501961590892


# Create a figure with two subplots side by side
fig, (ax1, ax2) = plt.subplots(1, 2)

# Plot 1: Histogram with KDE
sns.histplot(data=df, x='no_of_employees', kde=True, ax=ax1)
ax1.set_title('Number of Employees Distribution with KDE')
ax1.set_xlabel('Number of Employees')
ax1.set_ylabel('Count')

# Plot 2: Box Plot
sns.boxplot(y=df['no_of_employees'], ax=ax2)
ax2.set_title('Number of Employees Distribution (Box Plot)')
ax2.set_ylabel('Number of Employees')

plt.tight_layout()
plt.show()


# check if there are any -ve values
df['no_of_employees'].min()

-26


df['no_of_employees'].value_counts().max()

18


# Log Transformation (Strategy to handle Skewed Data)
plt.figure(figsize=(15, 6))

# Log transformation (adding 1 to handle zeros/negative values)
# NOTE: clip() is used to clip the values to a minimum of 0, as log works only for positive values
log_employees = np.log1p(df['no_of_employees'].clip(lower=0))

# Plot histogram with KDE
sns.histplot(log_employees, kde=True)

# find the reference point for number having highest frequency
df['no_of_employees'].value_counts().idxmax()

# Add reference lines with annotations
reference_points = [10, 100, 500, 1000, 2500, 5000, 10000]
for point in reference_points:
    log_point = np.log1p(point)
    plt.axvline(x=log_point, color='red', linestyle='--', alpha=0.7)
    vertical_offset = np.random.uniform(0.4, 0.9)
    plt.text(log_point + 0.1, plt.ylim()[1]*vertical_offset, f'{point}', verticalalignment='top')

# Customize x-axis ticks to show original values
log_ticks = [np.log1p(x) for x in [0, 10, 100, 1000, 10000, 100000]]
plt.xticks(log_ticks, ['0', '10', '100', '1K', '10K', '100K'])

plt.title('Log-transformed Number of Employees')
plt.xlabel('Log(Number of Employees + 1)')
plt.ylabel('Count')
plt.show()


# skewness and kurtosis for log-transformed data
print('Skewness : ', log_employees.skew())
print('Kurtosis : ', log_employees.kurt())

Skewness :  -0.09672037380703381
Kurtosis :  3.7313228285861375


# Binning (Strategy to handle Skewed Data)

# Create company size categories
bins = [0, 10, 100, 1000, 10000, 100000, float("inf")]
labels = [
    "Micro (<10)",
    "Very Small (10-99)",
    "Small (100-999)",
    "Medium (1K-10K)",
    "Large (10K-100K)",
    "Very Large (>100K)",
]

# Create new binned feature
company_size_feature = pd.cut(
    df["no_of_employees"].clip(lower=0), bins=bins, labels=labels
)

# Plot the distribution
plt.figure(figsize=(15, 6))
ax = sns.countplot(x=company_size_feature)
plt.title("Company Size Categories")
plt.xlabel("Company Size")
plt.ylabel("Count")
plt.xticks(rotation=45)

# Add count labels
for p in ax.patches:
    ax.annotate(
        f"{int(p.get_height())}",
        (p.get_x() + p.get_width() / 2.0, p.get_height()),
        ha="center",
        va="bottom",
    )

plt.tight_layout()
plt.show()


company_size_feature.value_counts()

Medium (1K-10K)       17803
Small (100-999)        5675
Large (10K-100K)       1165
Very Small (10-99)      533
Very Large (>100K)      271
Micro (<10)               0
Name: no_of_employees, dtype: int64


company_size_feature.value_counts(normalize=True) * 100

Medium (1K-10K)      69.96
Small (100-999)      22.30
Large (10K-100K)      4.58
Very Small (10-99)    2.09
Very Large (>100K)    1.06
Micro (<10)           0.00
Name: no_of_employees, dtype: float64


pd.crosstab(company_size_feature, df['case_status'], normalize='index') * 100


colname = 'yr_of_estab'


tb_describe(df[colname])

                count     mean    std      min      25%      50%      75%      max
-----------  --------  -------  -----  -------  -------  -------  -------  -------
yr_of_estab  25480.00  1979.41  42.37  1800.00  1976.00  1997.00  2005.00  2016.00


# Histogram and Box Plot
# Create a figure with two subplots side by side
fig, (ax1, ax2) = plt.subplots(1, 2)

# Plot 1: Histogram with KDE
sns.histplot(data=df, x=colname, kde=True, ax=ax1)
ax1.set_title('Year of Establishment Distribution with KDE')
ax1.set_xlabel('Year of Establishment')
ax1.set_ylabel('Count')

# Plot 2: Box Plot
sns.boxplot(y=df[colname], ax=ax2)
ax2.set_title('Year of Establishment Distribution (Box Plot)')
ax2.set_ylabel('Year of Establishment')

plt.tight_layout()
plt.show()


# Skewness and Kurtosis
print('Skewness : ', df[colname].skew())
print('Kurtosis : ', df[colname].kurt())

Skewness :  -2.037301271676438
Kurtosis :  3.506956756052395


# Binning (Strategy to handle Skewed Data)

# Create company age categories
bins = [1800, 1950, 1975, 1995, 2005, 2016]
labels = ['Historic (1800-1950)', 'Post-war (1951-1975)',
          'Modern (1976-1995)', 'Recent (1996-2005)',
          'New (2006-2016)']

# Create new binned feature
company_age_feature = pd.cut(df['yr_of_estab'],
                                   bins=bins, labels=labels)

# Plot the distribution
plt.figure(figsize=(15, 6))
ax = sns.countplot(x=company_age_feature)
plt.title('Company Age Categories')
plt.xlabel('Company Age')
plt.ylabel('Count')
plt.xticks(rotation=45)

# Add count labels
for p in ax.patches:
    ax.annotate(f'{int(p.get_height())}',
                (p.get_x() + p.get_width()/2., p.get_height()),
                ha='center', va='bottom')

plt.tight_layout()
plt.show()


company_age_feature.value_counts()

Recent (1996-2005)      8000
New (2006-2016)         5700
Modern (1976-1995)      5454
Historic (1800-1950)    4084
Post-war (1951-1975)    2226
Name: yr_of_estab, dtype: int64


company_age_feature.value_counts(normalize=True) * 100

Recent (1996-2005)     31.42
New (2006-2016)        22.38
Modern (1976-1995)     21.42
Historic (1800-1950)   16.04
Post-war (1951-1975)    8.74
Name: yr_of_estab, dtype: float64


pd.crosstab(company_age_feature, df['case_status'], normalize='index') * 100


# Binning (Strategy to handle Skewed Data)

# Create company age categories
bins = [1800, 1950, 1980, 2000, 2016]
labels = [
    "Legacy (Before 1950)",
    "Traditional Era (1950-1979)",
    "Growth Phase (1980-1999)",
    "New Age (2000-2016)",
]

# Create new binned feature
company_era_feature = pd.cut(df["yr_of_estab"], bins=bins, labels=labels)

# Plot the distribution
plt.figure(figsize=(15, 6))
ax = sns.countplot(x=company_era_feature)
plt.title("Company Era Categories")
plt.xlabel("Company Era")
plt.ylabel("Count")
plt.xticks(rotation=45)

# Add count labels
for p in ax.patches:
    ax.annotate(
        f"{int(p.get_height())}",
        (p.get_x() + p.get_width() / 2.0, p.get_height()),
        ha="center",
        va="bottom",
    )

plt.tight_layout()
plt.show()


company_era_feature.value_counts()

New Age (2000-2016)            9663
Growth Phase (1980-1999)       8460
Legacy (Before 1950)           4084
Traditional Era (1950-1979)    3257
Name: yr_of_estab, dtype: int64


company_era_feature.value_counts(normalize=True) * 100

New Age (2000-2016)           37.95
Growth Phase (1980-1999)      33.22
Legacy (Before 1950)          16.04
Traditional Era (1950-1979)   12.79
Name: yr_of_estab, dtype: float64


pd.crosstab(company_era_feature, df['case_status'], normalize='index') * 100


coname = 'region_of_employment'


df[coname].describe()

count         25480
unique            5
top       Northeast
freq           7195
Name: region_of_employment, dtype: object


df[coname].value_counts()

Northeast    7195
South        7017
West         6586
Midwest      4307
Island        375
Name: region_of_employment, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, coname, label='Region of Employment')


colname = 'prevailing_wage'


tb_describe(df[colname])

                    count      mean       std    min       25%       50%        75%        max
---------------  --------  --------  --------  -----  --------  --------  ---------  ---------
prevailing_wage  25480.00  74455.81  52815.94   2.14  34015.48  70308.21  107735.51  319210.27


# Skewness and Kurtosis
print('Skewness : ', df[colname].skew())
print('Kurtosis : ', df[colname].kurt())

Skewness :  0.7557760603032042
Kurtosis :  0.824273521052552


# Histogram and Box Plot
fig, (ax1, ax2) = plt.subplots(1, 2)

sns.histplot(data=df, x=colname, kde=True, ax=ax1)
ax1.set_title('Prevailing Wage Distribution with KDE')
ax1.set_xlabel('Prevailing Wage')
ax1.set_ylabel('Count')

sns.boxplot(y=df[colname], ax=ax2)
ax2.set_title('Prevailing Wage Distribution (Box Plot)')
ax2.set_ylabel('Prevailing Wage')
plt.tight_layout()
plt.show()


df[colname].describe()

count    25480.00
mean     74455.81
std      52815.94
min          2.14
25%      34015.48
50%      70308.21
75%     107735.51
max     319210.27
Name: prevailing_wage, dtype: float64


# Binning (Just to explore if it can be useful)

# Create wage categories based on observed distribution patterns
bins = [0, 40000, 80000, 150000, 320000]
labels = ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)',
          'Professional ($80K-150K)', 'Executive (>$150K)']

# Create new binned feature
wage_category_feature = pd.cut(df['prevailing_wage'],
                            bins=bins, labels=labels)

# Plot the distribution
plt.figure(figsize=(15, 6))
ax = sns.countplot(x=wage_category_feature)
plt.title('Wage Categories')
plt.xlabel('Wage Level')
plt.ylabel('Count')
plt.xticks(rotation=45)

# Add count labels
for p in ax.patches:
    ax.annotate(f'{int(p.get_height())}',
                (p.get_x() + p.get_width()/2., p.get_height()),
                ha='center', va='bottom')

plt.tight_layout()
plt.show()


wage_category_feature.value_counts()

Professional ($80K-150K)    9183
Entry-level ($0-40K)        7294
Mid-level ($40K-80K)        7165
Executive (>$150K)          1838
Name: prevailing_wage, dtype: int64


wage_category_feature.value_counts(normalize=True) * 100

Professional ($80K-150K)   36.04
Entry-level ($0-40K)       28.63
Mid-level ($40K-80K)       28.12
Executive (>$150K)          7.21
Name: prevailing_wage, dtype: float64


pd.crosstab(wage_category_feature, df['case_status'], normalize='index') * 100


colname = 'unit_of_wage'


df[colname].describe()

count     25480
unique        4
top        Year
freq      22962
Name: unit_of_wage, dtype: object


df[colname].value_counts()

Year     22962
Hour      2157
Week       272
Month       89
Name: unit_of_wage, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, colname, label='Unit of Wage')


colname = 'full_time_position'


df[colname].describe()

count     25480
unique        2
top           Y
freq      22773
Name: full_time_position, dtype: object


df[colname].value_counts()

Y    22773
N     2707
Name: full_time_position, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, colname, label='Full Time Position')


colname = 'case_status'


df[colname].describe()

count         25480
unique            2
top       Certified
freq          17018
Name: case_status, dtype: object


df[colname].value_counts()

Certified    17018
Denied        8462
Name: case_status, dtype: int64


plt.figure(figsize=(15, 6))
draw_countplot(df, colname, label='Case Status')


def create_stacked_bar(
    df,
    feature,
    title="Approval Rates by Feature",
    xlabel="Feature",
    ylabel="Percentage",
    colors=["steelblue", "lightcoral"],
):
    """
    Create a stacked bar chart showing the distribution of a binary target variable
    across categories of a feature.

    returns crosstab
    """
    target="case_status"

    # Create a cross-tabulation
    crosstab = pd.crosstab(df[feature], df[target], normalize="index") * 100

    # Create stacked bar chart
    plt.figure(figsize=(12, 8))
    crosstab.plot(kind="bar", stacked=True, color=colors, figsize=(12, 8))

    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    plt.xticks(rotation=45)
    plt.legend(title=target)

    # Add percentage labels on bars - simplified for binary target
    for i, category in enumerate(crosstab.index):
        # Get percentages
        first_value = crosstab.iloc[i, 0]  # First class (e.g., Certified)
        second_value = crosstab.iloc[i, 1]  # Second class (e.g., Denied)

        # Add text for first class (middle of its section)
        plt.text(
            i,
            first_value / 2,
            f"{first_value:.1f}%",
            ha="center",
            va="center",
            color="white",
            fontweight="bold",
        )

        # Add text for second class (middle of its section)
        plt.text(
            i,
            first_value + second_value / 2,
            f"{second_value:.1f}%",
            ha="center",
            va="center",
            color="black",
            fontweight="bold",
        )

    plt.tight_layout()
    plt.show()

    return crosstab


# Mosaic Plot (Helps to show both Dominant Category + Dominant Split/Imbalance)
def create_mosaic_plot(df, feature, title=None):
    """
    Create a mosaic plot showing relationship between a feature and target variable
    """
    target='case_status'
    # Create figure
    plt.figure(figsize=(12, 8))

    # Set default title if none provided
    if title is None:
        title = f'{feature} vs {target}'

    # Create the data dictionary for mosaic
    # This creates a dictionary with keys as tuples of (continent, case_status)
    mosaic_data = {(i, j): 0 for i in df[feature].unique() for j in df[target].unique()}
    for i, row in df.iterrows():
        mosaic_data[(row[feature], row[target])] += 1

    # Create mosaic plot
    mosaic(mosaic_data, gap=0.01, title=title,
        properties=lambda key: {'color': 'steelblue' if key[1] == 'Certified' else 'lightcoral'})

    plt.tight_layout()
    plt.show()


def perform_chi_square_test(df, feature1, feature2='case_status'):
    """
    Perform Chi-Square test of independence between two categorical variables

    Parameters:
    -----------
    df : pandas DataFrame
        The input DataFrame containing the data
    feature1 : str
        Name of first categorical feature
    feature2 : str
        Name of second categorical feature

    Returns:
    --------
    None
        Displays plots and prints test results
    """
    # Step 1: Create contingency table
    ct = pd.crosstab(df[feature1], df[feature2])

    # Step 2: Perform Chi-Square Test
    chi2, p, dof, expected = chi2_contingency(ct)

    # Step 3: Convert expected values to DataFrame for visualization
    expected_df = pd.DataFrame(expected, index=ct.index, columns=ct.columns)

    # Step 4: Plot heatmap (Observed vs. Expected)
    fig, axes = plt.subplots(1, 2, figsize=(12, 5))

    # Observed values heatmap
    sns.heatmap(ct, annot=True, cmap="Blues", fmt="d", ax=axes[0])
    axes[0].set_title(f"Observed Frequencies: {feature1} vs {feature2}")

    # Expected values heatmap
    sns.heatmap(expected_df, annot=True, cmap="Reds", fmt=".1f", ax=axes[1])
    axes[1].set_title(f"Expected Frequencies: {feature1} vs {feature2}")

    plt.tight_layout()
    plt.show()

    # Step 5: Print Chi-Square Test result
    print(f"Chi-Square Statistic: {chi2:.2f}")
    print(f"P-value: {p:.4f}")

    # Decision based on p-value
    alpha = 0.05  # Significance level
    if p < alpha:
        print(f"Reject H₀: There is a significant relationship between {feature1} and {feature2}")
    else:
        print(f"Fail to reject H₀: No significant relationship found between {feature1} and {feature2}")


# Grouped Bar Plot

def create_grouped_bar(df, feature1, feature2='case_status', title=None, sortAsc=None):
    """
    Create a grouped bar plot comparing a feature against case status with counts and percentages.
    """
    # Create the grouped bar plot
    # Sort values by count
    order = None
    if sortAsc is not None:
        order = df[feature1].value_counts(ascending=sortAsc).index
    ax = sns.countplot(data=df, x=feature1, hue=feature2, order=order)

    # Calculate percentages
    total = len(df)
    for p in ax.patches:
        percentage = p.get_height() / total * 100

        # Add count on top
        ax.annotate(f'{int(p.get_height())}',
                   (p.get_x() + p.get_width()/2., p.get_height()),
                   ha='center', va='bottom')

        # Add percentage inside bar
        ax.annotate(f'{percentage:.1f}%',
                   (p.get_x() + p.get_width()/2., p.get_height()/2),
                   ha='center', va='center')

    # Set title - use default if none provided
    if title is None:
        title = f'Case Status Distribution by {feature1}'
    plt.title(title)

    plt.xlabel(snake_to_pascal(feature1))
    plt.ylabel(snake_to_pascal(feature2))

    # plt.xticks(rotation=45)
    plt.show()


# Contigencey Table
def create_contingency_table(df, feature1, feature2='case_status', pct=False):
    """
    Create a contingency table for two categorical features showing both counts and percentages.
    """
    return pd.crosstab(df[feature1], df[feature2], normalize= 'index' if pct else False) * 100


# Grouped Bar Plot
sns.countplot(data=df, x='continent', hue='case_status')
plt.title('Case Status Distribution by Continent')
plt.xticks(rotation=45)
plt.show()


ct = create_stacked_bar(df, 'continent', 'Visa Approval Rates by Continent', 'Continent', 'Percentage')

<Figure size 1200x800 with 0 Axes>


# Grouped Bar Plot
sns.countplot(data=df, x='has_job_experience', hue='case_status')
plt.title('Case Status Distribution by Has Job Experience')
plt.xticks(rotation=45)
plt.show()


create_mosaic_plot(df, 'has_job_experience', title='Visa Approval Rates by Has Job Experience')

<Figure size 1200x800 with 0 Axes>


ct = create_stacked_bar(df, 'has_job_experience', 'Visa Approval Rates by Has Job Experience', 'Has Job Experience', 'Percentage')

<Figure size 1200x800 with 0 Axes>


create_contingency_table(df, 'education_of_employee')


create_mosaic_plot(df, 'education_of_employee', title='Visa Approval Rates by Education')

<Figure size 1200x800 with 0 Axes>


create_grouped_bar(df, 'education_of_employee', sortAsc=True)


perform_chi_square_test(df, 'education_of_employee')

Chi-Square Statistic: 2772.21
P-value: 0.0000
Reject H₀: There is a significant relationship between education_of_employee and case_status


# Now split by job experience
g = sns.catplot(data=df, x='education_of_employee', hue='case_status', col='has_job_experience',
               kind='count', palette=['steelblue', 'lightcoral'],
               order=['High School', 'Bachelor\'s', 'Master\'s', 'Doctorate'],
               height=6, aspect=1.2)
g.figure.suptitle('Visa Outcomes by Education and Job Experience', y=1.05)
g.set_axis_labels('Education Level', 'Count')
g.set_titles('Job Experience: {col_name}')
plt.tight_layout()
plt.show()


# Violin plot for Year of Establishment by Case Status
plt.figure(figsize=(12, 6))
sns.violinplot(x='case_status', y='yr_of_estab', data=df)
plt.title('Distribution of Company Establishment Year by Visa Status')
plt.xlabel('Case Status')
plt.ylabel('Year of Establishment')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.show()


# Get summary statistics for each case status
for status in df['case_status'].unique():
    subset = df[df['case_status'] == status]['yr_of_estab']
    print(f"\nStatistics for {status}:")
    print(f"Mean: {subset.mean():.2f}")
    print(f"Median: {subset.median():.2f}")
    print(f"25th percentile: {subset.quantile(0.25):.2f}")
    print(f"75th percentile: {subset.quantile(0.75):.2f}")
    print(f"Min: {subset.min():.2f}")
    print(f"Max: {subset.max():.2f}")
    print(f"Skewness: {subset.skew():.2f}")

Statistics for Denied:
Mean: 1978.89
Median: 1997.00
25th percentile: 1975.00
75th percentile: 2004.00
Min: 1800.00
Max: 2016.00
Skewness: -2.04

Statistics for Certified:
Mean: 1979.67
Median: 1997.00
25th percentile: 1976.00
75th percentile: 2005.00
Min: 1800.00
Max: 2016.00
Skewness: -2.03


# KDE plot for Year of Establishment by Case Status
plt.figure(figsize=(12, 6))
sns.kdeplot(data=df, x='yr_of_estab', hue='case_status',
            palette=['steelblue', 'lightcoral'],
            common_norm=False, fill=True, alpha=0.5)
plt.title('Density Distribution of Company Establishment Year by Visa Status')
plt.xlabel('Year of Establishment')
plt.ylabel('Density')
plt.grid(axis='y', linestyle='--', alpha=0.7)

# Add vertical lines for medians without adding to legend
for i, (status, color) in enumerate(zip(['Certified', 'Denied'], ['steelblue', 'lightcoral'])):
    median = df[df['case_status'] == status]['yr_of_estab'].median()
    plt.axvline(x=median, color=color, linestyle='--')

    # Stagger the text positions vertically
    vertical_position = plt.ylim()[1] * (0.9 - i*0.1)  # Different heights
    horizontal_offset = -32  # Left side

    # Add text annotation
    plt.text(median + horizontal_offset, vertical_position,
             f'{status} Median: {median}',
             color=color, fontweight='bold')

plt.xlim(1900, 2020)  # Focus on more recent years
#plt.legend(title='Case Status')
plt.show()


# Violin plots
plt.figure(figsize=(14, 8))
sns.violinplot(x='continent', y='yr_of_estab', hue='case_status',
               data=df, palette=['steelblue', 'lightcoral'], split=True)
plt.title('Company Establishment Year Distribution by Continent and Visa Status')
plt.xlabel('Continent')
plt.ylabel('Year of Establishment')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.legend(title='Case Status')
plt.xticks(rotation=45)
plt.show()


# Box plot with better handling of skewed data
plt.figure(figsize=(12, 6))
# Clip negative values and apply log transformation
transformed_employees = np.log1p(df['no_of_employees'].clip(lower=0))
sns.boxplot(x=df['case_status'], y=transformed_employees)
plt.title('Log-Transformed Number of Employees by Visa Status')
plt.xlabel('Case Status')
plt.ylabel('Log(Number of Employees + 1)')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.show()


# KDE plot for log-transformed number of employees by case status
plt.figure(figsize=(12, 6))
# Applying Log transformation so as to get better viz (otherwise tree models dont need this during modeling)
# Clip negative values and apply log transformation
transformed_employees = np.log1p(df['no_of_employees'].clip(lower=0))

# Create KDE plot
sns.kdeplot(data=df, x=transformed_employees, hue='case_status',
            palette=['steelblue', 'lightcoral'],
            common_norm=False, fill=True, alpha=0.5)

plt.title('Distribution of Log-Transformed Number of Employees by Visa Status')
plt.xlabel('Log(Number of Employees + 1)')
plt.ylabel('Density')
plt.grid(axis='y', linestyle='--', alpha=0.7)
#plt.legend(title='Case Status')
plt.show()


create_contingency_table(df, 'region_of_employment')


create_contingency_table(df, 'region_of_employment', pct=True)


create_mosaic_plot(df, 'region_of_employment', title='Visa Approval Rates by Region of Employment')

<Figure size 1200x800 with 0 Axes>


perform_chi_square_test(df, 'region_of_employment')

Chi-Square Statistic: 298.45
P-value: 0.0000
Reject H₀: There is a significant relationship between region_of_employment and case_status


plt.figure(figsize=(12, 6))
sns.boxplot(data=df, x="region_of_employment", y="yr_of_estab", hue='case_status')
plt.xticks(rotation=45)  # Rotate x-axis labels for better readability
plt.title("Year of Establishment Distribution Across Employment Regions")
plt.show()


# Box plot for Prevailing Wage by Case Status
plt.figure(figsize=(12, 6))
sns.boxplot(x='case_status', y='prevailing_wage', data=df)
plt.title('Prevailing Wage Distribution by Visa Status')
plt.xlabel('Case Status')
plt.ylabel('Prevailing Wage')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.show()


# KDE plot for prevailing wage by case status
plt.figure(figsize=(12, 6))

# Create separate KDE plots for each case status
for status, color in zip(['Certified', 'Denied'], ['steelblue', 'lightcoral']):
    subset = df[df['case_status'] == status]['prevailing_wage']
    sns.kdeplot(x=subset, label=status, color=color, fill=True, alpha=0.5)

plt.title('Prevailing Wage Distribution by Visa Status')
plt.xlabel('Prevailing Wage')
plt.ylabel('Density')
plt.grid(axis='y', linestyle='--', alpha=0.7)
plt.legend(title='Case Status')
plt.show()


plt.figure(figsize=(12, 6))
sns.boxplot(data=df, x="region_of_employment", y="prevailing_wage", hue='case_status')
plt.xticks(rotation=45)  # Rotate x-axis labels for better readability
plt.title("Prevailing Wage Distribution Across Employment Regions")
plt.show()


create_contingency_table(df, 'unit_of_wage')


create_contingency_table(df, 'unit_of_wage', pct=True)


create_mosaic_plot(df, 'unit_of_wage', title='Visa Approval Rates by Unit of Wage')

<Figure size 1200x800 with 0 Axes>


perform_chi_square_test(df, 'unit_of_wage')

Chi-Square Statistic: 1108.51
P-value: 0.0000
Reject H₀: There is a significant relationship between unit_of_wage and case_status


create_grouped_bar(df, 'full_time_position', sortAsc=True)


create_mosaic_plot(df, 'full_time_position', title='Visa Approval Rates by Full Time Position')

<Figure size 1200x800 with 0 Axes>


create_contingency_table(df, 'full_time_position', pct=True)


#Why (Good Practice)
# ? -> Clear seperation between EDA & Preprocessing (Phases)
#1. Any accidental modifications made during EDA won't affect your preprocessing
#2. starting with clean data for the preprocessing phase
df = df_original.copy()


# find columns with negative values
numeric_cols = df.select_dtypes(include=['int64', 'float64']).columns


negative_cols = [col for col in numeric_cols if df[col].min() < 0]
print(negative_cols)

['no_of_employees']


# Convert negative employee counts to absolute values
print(f"Number of negative employee counts before: {(df['no_of_employees'] < 0).sum()}")
df['no_of_employees'] = df['no_of_employees'].abs()
print(f"Number of negative employee counts after: {(df['no_of_employees'] < 0).sum()}")

Number of negative employee counts before: 33
Number of negative employee counts after: 0


# Outlier Detection using IQR method and Z-Score

outlier_stats = {}
for col in numeric_cols:
    # IQR method
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower_bound_iqr = Q1 - 1.5 * IQR
    upper_bound_iqr = Q3 + 1.5 * IQR
    outliers_iqr = df[(df[col] < lower_bound_iqr) | (df[col] > upper_bound_iqr)][col]

    # Z-Score method
    #***
    # Why abs()
    # It's required to take the absolute value of z-scores when detecting outliers.
    # This is because outliers can be
    # either unusually high (positive z-score) or unusually low (negative z-score),
    # and we're interested in the magnitude of deviation from the mean, not the direction.
    #***
    z_scores = np.abs(zscore(df[col]))
    zscore_outlier_count = np.sum(z_scores > 3)
    zscore_outlier_percentage = (zscore_outlier_count / len(df)) * 100

    outlier_stats[col] = {
        'min': df[col].min(),
        'max': df[col].max(),
        'iqr_outlier_count': len(outliers_iqr),
        'iqr_outlier_percentage': (len(outliers_iqr) / len(df)) * 100,
        'zscore_outlier_count': zscore_outlier_count,
        'zscore_outlier_percentage': zscore_outlier_percentage,
    }

# Display outlier statistics
for col, stats in outlier_stats.items():
    print(f"\n{col}:")
    print(f"IQR Method - Number of outliers: {stats['iqr_outlier_count']}")
    print(f"IQR Method - Percentage of outliers: {stats['iqr_outlier_percentage']:.2f}%")
    print(f"Z-Score Method - Number of outliers: {stats['zscore_outlier_count']}")
    print(f"Z-Score Method - Percentage of outliers: {stats['zscore_outlier_percentage']:.2f}%")
    print(f"Range: {stats['min']} to {stats['max']}")

no_of_employees:
IQR Method - Number of outliers: 1556
IQR Method - Percentage of outliers: 6.11%
Z-Score Method - Number of outliers: 429
Z-Score Method - Percentage of outliers: 1.68%
Range: 11 to 602069

yr_of_estab:
IQR Method - Number of outliers: 3260
IQR Method - Percentage of outliers: 12.79%
Z-Score Method - Number of outliers: 695
Z-Score Method - Percentage of outliers: 2.73%
Range: 1800 to 2016

prevailing_wage:
IQR Method - Number of outliers: 427
IQR Method - Percentage of outliers: 1.68%
Z-Score Method - Number of outliers: 294
Z-Score Method - Percentage of outliers: 1.15%
Range: 2.1367 to 319210.27


# Total columns in df (before binning)
print('Total columns in df (before binning) = ', len(df.columns))

Total columns in df (before binning) =  12


# Create company size categories
bins = [0, 50, 250, 1000, 10000, float("inf")]
labels = [
    "Micro (1-50)",
    "Small (51-250)",
    "Medium (251-1000)",
    "Large (1001-10000)",
    "Enterprise (>10000)"
]

# Create new binned feature
df['company_size'] = pd.cut(
    # Handle negative values (to be on safe side)
    df["no_of_employees"].clip(lower=0),
    bins=bins,
    labels=labels
)

# Check the distribution
df['company_size'].value_counts()

Large (1001-10000)     17803
Medium (251-1000)       4702
Enterprise (>10000)     1436
Small (51-250)          1290
Micro (1-50)             249
Name: company_size, dtype: int64


# check if summ of all counts is equal to total rows
print(df["company_size"].value_counts().sum())
print(len(df))

25480
25480


# print few rows no_of_employees & company_size
df[['no_of_employees', 'company_size']].head()


# min and max of yr_of_estab
print(df["yr_of_estab"].min())
print(df["yr_of_estab"].max())

1800
2016


# Create company era categories
# *?
# NOTE: for any 2 point in bins bin is decided like for example
# Assuming x-1 and z are extreme points of your dataset
#  [x, y, z] then
#  - bin 1 is (x, y]
#  - bin 2 is (y, z]
# **
# Why 1799 -> 1800 - 1  // so that 1800 gets acounted
# End 2016 -> becasuse end is inclusive
bins = [1799, 1949, 1979, 1999, 2010, 2016]
labels = [
    "Legacy (Before 1950)",
    "Traditional Era (1950-1979)",
    "Growth Phase (1980-1999)",
    "Digital Transition Era (2000-2010)",
    "AI & Mobile Era (2011-2016)",
]

# Create new binned feature
df["company_era"] = pd.cut(
    df["yr_of_estab"],
    bins=bins,
    labels=labels,
)

# Check the distribution
df["company_era"].value_counts()

Digital Transition Era (2000-2010)    8664
Growth Phase (1980-1999)              7825
Legacy (Before 1950)                  4052
Traditional Era (1950-1979)           3134
AI & Mobile Era (2011-2016)           1805
Name: company_era, dtype: int64


# check if summ of all counts is equal to total rows
print(df["company_era"].value_counts().sum())
print(len(df))

25480
25480


# print few rows yr_of_estab & company_era
df[["yr_of_estab", "company_era"]].head()


# check for year of estab 1800
df[df["yr_of_estab"] == 1800].head()


# Let see mosaic plot to check dominance of category and splitting ability at high level
create_mosaic_plot(df, 'company_era', title='Visa Approval Rates by Company Era')

<Figure size 1200x800 with 0 Axes>


# min and max of prevailing_wage
print(df["prevailing_wage"].min())
print(df["prevailing_wage"].max())

2.1367
319210.27


# Create wage categories
bins = [0, 40_000, 80_000, 150_000, 3_20_000]
labels = [
    "Entry-level ($0-40K)",
    "Mid-level ($40K-80K)",
    "Professional ($80K-150K)",
    "Executive (>$150K)",
]
# Create new binned feature
df["wage_category"] = pd.cut(df["prevailing_wage"], bins=bins, labels=labels)


# Check the distribution
print("Distribution of wage categories:")
print(df['wage_category'].value_counts())

print("\nPercentage distribution:")
print(df['wage_category'].value_counts(normalize=True) * 100)

Distribution of wage categories:
Professional ($80K-150K)    9183
Entry-level ($0-40K)        7294
Mid-level ($40K-80K)        7165
Executive (>$150K)          1838
Name: wage_category, dtype: int64

Percentage distribution:
Professional ($80K-150K)   36.04
Entry-level ($0-40K)       28.63
Mid-level ($40K-80K)       28.12
Executive (>$150K)          7.21
Name: wage_category, dtype: float64


# show few rows of prevailing_wage & wage_category
df[["prevailing_wage", "wage_category"]].head()


# check if summ of all counts is equal to total rows
print(df["wage_category"].value_counts().sum())
print(len(df))

25480
25480


# Check relationship with target variable
perform_chi_square_test(df, 'wage_category')

Chi-Square Statistic: 165.58
P-value: 0.0000
Reject H₀: There is a significant relationship between wage_category and case_status


df['case_status'].value_counts()

Certified    17018
Denied        8462
Name: case_status, dtype: int64


df['case_status'].unique()

array(['Denied', 'Certified'], dtype=object)


df['case_status'].value_counts()

Certified    17018
Denied        8462
Name: case_status, dtype: int64


df['case_status'].dtype  # object/string

dtype('O')


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 25480 entries, 0 to 25479
Data columns (total 15 columns):
 #   Column                 Non-Null Count  Dtype   
---  ------                 --------------  -----   
 0   case_id                25480 non-null  object  
 1   continent              25480 non-null  object  
 2   education_of_employee  25480 non-null  object  
 3   has_job_experience     25480 non-null  object  
 4   requires_job_training  25480 non-null  object  
 5   no_of_employees        25480 non-null  int64   
 6   yr_of_estab            25480 non-null  int64   
 7   region_of_employment   25480 non-null  object  
 8   prevailing_wage        25480 non-null  float64 
 9   unit_of_wage           25480 non-null  object  
 10  full_time_position     25480 non-null  object  
 11  case_status            25480 non-null  object  
 12  company_size           25480 non-null  category
 13  company_era            25480 non-null  category
 14  wage_category          25480 non-null  category
dtypes: category(3), float64(1), int64(2), object(9)
memory usage: 2.4+ MB


# In-Place Target Encoding

# Encode target variable: Certified -> 1, Denied -> 0
df['case_status'] = df['case_status'].map({'Certified': 1, 'Denied': 0})

# Verify the encoding
print(df['case_status'].value_counts())

1    17018
0     8462
Name: case_status, dtype: int64


df['case_status'].dtype

dtype('int64')


print('Total columns in df (after binning) = ', len(df.columns))

Total columns in df (after binning) =  15


# drop case_id
df.drop('case_id', axis=1, inplace=True)


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 25480 entries, 0 to 25479
Data columns (total 14 columns):
 #   Column                 Non-Null Count  Dtype   
---  ------                 --------------  -----   
 0   continent              25480 non-null  object  
 1   education_of_employee  25480 non-null  object  
 2   has_job_experience     25480 non-null  object  
 3   requires_job_training  25480 non-null  object  
 4   no_of_employees        25480 non-null  int64   
 5   yr_of_estab            25480 non-null  int64   
 6   region_of_employment   25480 non-null  object  
 7   prevailing_wage        25480 non-null  float64 
 8   unit_of_wage           25480 non-null  object  
 9   full_time_position     25480 non-null  object  
 10  case_status            25480 non-null  int64   
 11  company_size           25480 non-null  category
 12  company_era            25480 non-null  category
 13  wage_category          25480 non-null  category
dtypes: category(3), float64(1), int64(3), object(7)
memory usage: 2.2+ MB


# Check the total number of rows
df.shape

(25480, 14)

encoder.fit_transform(df[['education_of_employee']])  # Double brackets create a 2D array


ordinal_cols = [
    'education_of_employee',
    'company_size',
    'company_era',
    'wage_category'
]


# Define Categorical Order for Each Column that needs Ordinal Encoding
education_of_employee_cats = []
company_size_cats = []
company_era_cats = []
wage_category_cats = []


# Get unique values in their current order
for col in ordinal_cols:
    categories = df[col].unique()
    print(f"\nCurrent unique categories for {col}:")
    for cat in categories:
        print(f"- {cat}")

Current unique categories for education_of_employee:
- High School
- Master's
- Bachelor's
- Doctorate

Current unique categories for company_size:
- Enterprise (>10000)
- Large (1001-10000)
- Small (51-250)
- Medium (251-1000)
- Micro (1-50)

Current unique categories for company_era:
- Digital Transition Era (2000-2010)
- Legacy (Before 1950)
- AI & Mobile Era (2011-2016)
- Growth Phase (1980-1999)
- Traditional Era (1950-1979)

Current unique categories for wage_category:
- Entry-level ($0-40K)
- Professional ($80K-150K)
- Mid-level ($40K-80K)
- Executive (>$150K)


df['education_of_employee'].unique()

array(['High School', "Master's", "Bachelor's", 'Doctorate'], dtype=object)


education_of_employee_cats = [
    'High School',
    "Bachelor's",
    "Master's",
    'Doctorate'
]


encoder = OrdinalEncoder(categories=[education_of_employee_cats])
transformed = encoder.fit_transform(df[['education_of_employee']]).ravel()


df['education_of_employee'].value_counts()

Bachelor's     10234
Master's        9634
High School     3420
Doctorate       2192
Name: education_of_employee, dtype: int64


# Get unique values and their counts
pd.Series(transformed).value_counts()

1.00    10234
2.00     9634
0.00     3420
3.00     2192
dtype: int64


df['company_size'].unique()

['Enterprise (>10000)', 'Large (1001-10000)', 'Small (51-250)', 'Medium (251-1000)', 'Micro (1-50)']
Categories (5, object): ['Micro (1-50)' < 'Small (51-250)' < 'Medium (251-1000)' < 'Large (1001-10000)' < 'Enterprise (>10000)']


company_size_cats = [
    'Micro (1-50)',
    'Small (51-250)',
    'Medium (251-1000)',
    'Large (1001-10000)',
    'Enterprise (>10000)'
]
encoder = OrdinalEncoder(categories=[company_size_cats])
transformed = encoder.fit_transform(df[['company_size']]).ravel()


df['company_size'].value_counts()

Large (1001-10000)     17803
Medium (251-1000)       4702
Enterprise (>10000)     1436
Small (51-250)          1290
Micro (1-50)             249
Name: company_size, dtype: int64


pd.Series(transformed).value_counts()

3.00    17803
2.00     4702
4.00     1436
1.00     1290
0.00      249
dtype: int64


df['company_era'].unique()

['Digital Transition Era (2000-2010)', 'Legacy (Before 1950)', 'AI & Mobile Era (2011-2016)', 'Growth Phase (1980-1999)', 'Traditional Era (1950-1979)']
Categories (5, object): ['Legacy (Before 1950)' < 'Traditional Era (1950-1979)' < 'Growth Phase (1980-1999)' < 'Digital Transition Era (2000-2010)' < 'AI & Mobile Era (2011-2016)']


company_era_cats = [
    'Legacy (Before 1950)',
    'Traditional Era (1950-1979)',
    'Growth Phase (1980-1999)',
    'Digital Transition Era (2000-2010)',
    'AI & Mobile Era (2011-2016)',
]
encoder = OrdinalEncoder(categories=[company_era_cats])
transformed = encoder.fit_transform(df[['company_era']]).ravel()


df['company_era'].value_counts()

Digital Transition Era (2000-2010)    8664
Growth Phase (1980-1999)              7825
Legacy (Before 1950)                  4052
Traditional Era (1950-1979)           3134
AI & Mobile Era (2011-2016)           1805
Name: company_era, dtype: int64


pd.Series(transformed).value_counts()

3.00    8664
2.00    7825
0.00    4052
1.00    3134
4.00    1805
dtype: int64


df['wage_category'].unique()

['Entry-level ($0-40K)', 'Professional ($80K-150K)', 'Mid-level ($40K-80K)', 'Executive (>$150K)']
Categories (4, object): ['Entry-level ($0-40K)' < 'Mid-level ($40K-80K)' < 'Professional ($80K-150K)' < 'Executive (>$150K)']


wage_category_cats = [
    'Entry-level ($0-40K)',
    'Mid-level ($40K-80K)',
    'Professional ($80K-150K)',
    'Executive (>$150K)',
]
encoder = OrdinalEncoder(categories=[wage_category_cats])
transformed = encoder.fit_transform(df[['wage_category']]).ravel()


df['wage_category'].value_counts()

Professional ($80K-150K)    9183
Entry-level ($0-40K)        7294
Mid-level ($40K-80K)        7165
Executive (>$150K)          1838
Name: wage_category, dtype: int64


pd.Series(transformed).value_counts()

2.00    9183
0.00    7294
1.00    7165
3.00    1838
dtype: int64


# List of all ordinal columns categories
ordinal_cols_cats = [
    education_of_employee_cats,
    company_size_cats,
    company_era_cats,
    wage_category_cats
]


# You can’t say "hourly < weekly < monthly < yearly" in a strict sense
# because conversion depends on actual wages, not a fixed ranking
ohe_cols = [
    'continent',
    'region_of_employment',
    'unit_of_wage',
]


# let see 3 cols data
print(df['has_job_experience'].unique())
print(df['requires_job_training'].unique())
print(df['full_time_position'].unique())

['N' 'Y']
['N' 'Y']
['Y' 'N']


binary_cols = [
    'has_job_experience',
    'requires_job_training',
    'full_time_position'
]


binary_cats = ['N', 'Y']


# Why repeated 3 times ?
# Because OrdinalEncoder needs same shape as input matrix for param categories
# TODO: Find Good way to address this Binary Encoding Maybe CustomLabelEncoder class that overcomes current limitations of LabelEncoder can be used instead of ordinal encoding
binary_cols_cats = [
    binary_cats,
    binary_cats,
    binary_cats
]


encoder = OrdinalEncoder(categories=[binary_cats])
transformed = encoder.fit_transform(df[['has_job_experience']]).ravel()


df['has_job_experience'].value_counts()

Y    14802
N    10678
Name: has_job_experience, dtype: int64


# Verified ✔️
pd.Series(transformed).value_counts()

1.00    14802
0.00    10678
dtype: int64


encoder = OrdinalEncoder(categories=[binary_cats])
transformed = encoder.fit_transform(df[['requires_job_training']]).ravel()


df['requires_job_training'].value_counts()

N    22525
Y     2955
Name: requires_job_training, dtype: int64


pd.Series(transformed).value_counts()

0.00    22525
1.00     2955
dtype: int64


encoder = OrdinalEncoder(categories=[binary_cats])
transformed = encoder.fit_transform(df[['full_time_position']]).ravel()


df['full_time_position'].value_counts()

Y    22773
N     2707
Name: full_time_position, dtype: int64


pd.Series(transformed).value_counts()

1.00    22773
0.00     2707
dtype: int64


df.columns

Index(['continent', 'education_of_employee', 'has_job_experience',
       'requires_job_training', 'no_of_employees', 'yr_of_estab',
       'region_of_employment', 'prevailing_wage', 'unit_of_wage',
       'full_time_position', 'case_status', 'company_size', 'company_era',
       'wage_category'],
      dtype='object')


# Check total rows
df.shape

(25480, 14)


df['case_status'].value_counts()

1    17018
0     8462
Name: case_status, dtype: int64


# Separate features and target
X = df.drop('case_status', axis=1)
y = df['case_status']

# The stratify=y parameter ensures that the class distribution
# in the train and test sets matches the original distribution,
# which is important for imbalanced datasets.

# Create train-test split (80-20)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)


# Verify the split
print(f"Training set shape: {X_train.shape}")
print(f"Testing set shape: {X_test.shape}")
print(f"Training set distribution:\n{y_train.value_counts()}")
print(f"Testing set distribution:\n{y_test.value_counts()}")

Training set shape: (20384, 13)
Testing set shape: (5096, 13)
Training set distribution:
1    13614
0     6770
Name: case_status, dtype: int64
Testing set distribution:
1    3404
0    1692
Name: case_status, dtype: int64


# Function to compute different metrics to check performance of a classification model built using sklearn
def get_classification_metrics(y, y_pred):
    """
    Function to compute different metrics to check classification model performance

    Parameters:
        y: dependent variable/ground truth labels
        y_pred: predicted target values

    Returns:
        pd.DataFrame containing model performance metrics (Accuracy, Recall, Precision, F1)
    """
    # compute various metrics
    acc = accuracy_score(y, y_pred)
    recall = recall_score(y, y_pred)
    precision = precision_score(y, y_pred)
    f1 = f1_score(y, y_pred)
    # intentionally commented below 2 lines as we will focus on main 4 metrics once model is built
    #f1_weighted = f1_score(y, y_pred, average='weighted')
    #recall_weighted = recall_score(y, y_pred, average='weighted')

    # creating a dataframe of metrics
    df_perf = pd.DataFrame(
        {
            "Accuracy": acc,
            "Recall": recall,
            "Precision": precision,
            "F1": f1,
            #"F1_Weighted": f1_weighted,
            #"Recall_Weighted": recall_weighted
        },
        index=[0]
    )

    return df_perf


# Helper function to remove step prefix from pipeline's transformed column name
def remove_pipeline_step_prefix(name):
    # return name.replace('classifier__', '')
    return name.split("__")[-1]  # Removes transformer prefix


def get_feature_importances(model, columnNames):
    """
    Get feature importances from a trained model.

    Args:
        model: model
        columnNames: Name of the features

    Returns:
        DataFrame with feature names and their importance scores, sorted by importance
    """
    importances = pd.DataFrame({
        'feature': columnNames,
        'importance': model.feature_importances_
    }).sort_values('importance', ascending=False)

    return importances


def plot_feature_importance(importances, top_n=None):
    """
    Plot feature importances as a horizontal bar chart.

    Parameters:
        importances: DataFrame with 'feature' and 'importance' columns
        figsize: tuple, size of the figure (width, height)
        top_n: int, optional number of top features to show (shows all if None)
    """
    # Create copy to avoid modifying original
    df = importances.copy()

    # Limit to top N features if specified
    if top_n is not None:
        df = df.head(top_n)

    # Sort by importance
    df = df.sort_values('importance', ascending=False)

    sns.barplot(x='importance', y='feature', data=df)
    plt.title('Feature Importance')
    plt.tight_layout()
    plt.show()


# score map
# Model Results
results = dict()


X_train.columns

Index(['continent', 'education_of_employee', 'has_job_experience',
       'requires_job_training', 'no_of_employees', 'yr_of_estab',
       'region_of_employment', 'prevailing_wage', 'unit_of_wage',
       'full_time_position', 'company_size', 'company_era', 'wage_category'],
      dtype='object')


for col, cats in zip(ordinal_cols, ordinal_cols_cats):
    print(f"{col} -> {cats}")

education_of_employee -> ['High School', "Bachelor's", "Master's", 'Doctorate']
company_size -> ['Micro (1-50)', 'Small (51-250)', 'Medium (251-1000)', 'Large (1001-10000)', 'Enterprise (>10000)']
company_era -> ['Legacy (Before 1950)', 'Traditional Era (1950-1979)', 'Growth Phase (1980-1999)', 'Digital Transition Era (2000-2010)', 'AI & Mobile Era (2011-2016)']
wage_category -> ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)', 'Professional ($80K-150K)', 'Executive (>$150K)']


for col, cats in zip(binary_cols, binary_cols_cats):
    print(f"{col} -> {cats}")

has_job_experience -> ['N', 'Y']
requires_job_training -> ['N', 'Y']
full_time_position -> ['N', 'Y']


# Create the column transformer
preprocessor = ColumnTransformer(
    transformers=[
        ('ordinal', OrdinalEncoder(categories=ordinal_cols_cats), ordinal_cols),
        ('binary', OrdinalEncoder(categories=binary_cols_cats), binary_cols),
        ('onehot', OneHotEncoder(drop='first', sparse_output=False), ohe_cols)
    ],
    remainder='passthrough'  # Keeps numeric columns ['no_of_employees', 'yr_of_estab', 'prevailing_wage']
)


# Fit preprocessor on training data to get feature names
# This is only for feature interpretation purposes, not for actual transformation
preprocessor.fit(X_train)

# Get the transformed column names for later use in feature importance
transformed_feature_names = preprocessor.get_feature_names_out()

transformed_columns = [remove_pipeline_step_prefix(name) for name in transformed_feature_names]


print(pd.Series(transformed_columns).values)

['education_of_employee' 'company_size' 'company_era' 'wage_category'
 'has_job_experience' 'requires_job_training' 'full_time_position'
 'continent_Asia' 'continent_Europe' 'continent_North America'
 'continent_Oceania' 'continent_South America'
 'region_of_employment_Midwest' 'region_of_employment_Northeast'
 'region_of_employment_South' 'region_of_employment_West'
 'unit_of_wage_Month' 'unit_of_wage_Week' 'unit_of_wage_Year'
 'no_of_employees' 'yr_of_estab' 'prevailing_wage']


print('Original Columns Order')
print(X_train.columns)

Original Columns Order
Index(['continent', 'education_of_employee', 'has_job_experience',
       'requires_job_training', 'no_of_employees', 'yr_of_estab',
       'region_of_employment', 'prevailing_wage', 'unit_of_wage',
       'full_time_position', 'company_size', 'company_era', 'wage_category'],
      dtype='object')


X_train_transformed = preprocessor.fit_transform(X_train)
# convert to dataframe with original index (output from preprocessor will be nd array (numeric)
# not dataframe so we loose Row-Index and Column-Order both in Pipelineing)
# Recreating the transformed X_train dataframe with original index and transformed columns
X_train_transformed = pd.DataFrame(X_train_transformed,
                                 columns=transformed_columns,
                                 index=X_train.index)
X_train_transformed.head()


X_train.head()


print('\n'.join(transformed_columns))

education_of_employee
company_size
company_era
wage_category
has_job_experience
requires_job_training
full_time_position
continent_Asia
continent_Europe
continent_North America
continent_Oceania
continent_South America
region_of_employment_Midwest
region_of_employment_Northeast
region_of_employment_South
region_of_employment_West
unit_of_wage_Month
unit_of_wage_Week
unit_of_wage_Year
no_of_employees
yr_of_estab
prevailing_wage


# Total number of features after transformation
n_features = len(transformed_columns)

# All columns except the last 3 are categorical
categorical_features = list(range(n_features - 3))
categorical_features

[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18]


# Create oversampling step with SMOTENC
oversampler = SMOTENC(
    categorical_features=categorical_features,
    random_state=42,
    sampling_strategy='auto'  # Will balance classes by default
)


# Create undersampling step
undersampler = RandomUnderSampler(
    random_state=42,
    sampling_strategy='auto'  # Will balance classes by default
)


# Check class imbalance in train set
y_train.value_counts(normalize=True).mul(100).round(2)

1   66.79
0   33.21
Name: case_status, dtype: float64


# Create pipeline with preprocessing and default decision tree
dt_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', DecisionTreeClassifier(random_state=42, class_weight='balanced'))
])

# Fit the pipeline on training data
dt_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = dt_pipeline.predict(X_test)


dt_result = get_classification_metrics(y_test, y_pred)
dt_result


# register result
results['dt'] = dt_result


# Create pipeline with preprocessing, oversampling, and decision tree
dt_oversampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', oversampler),
    ('classifier', DecisionTreeClassifier(random_state=42))
])

# Fit the pipeline on training data
dt_oversampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = dt_oversampled_pipeline.predict(X_test)


dt_oversampled_result = get_classification_metrics(y_test, y_pred)
dt_oversampled_result


results['dt_up'] = dt_oversampled_result


# Create pipeline with preprocessing, undersampling, and decision tree
dt_undersampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', undersampler),
    ('classifier', DecisionTreeClassifier(random_state=42))
])

# Fit the pipeline on training data
dt_undersampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = dt_undersampled_pipeline.predict(X_test)


dt_undersampled_result = get_classification_metrics(y_test, y_pred)
dt_undersampled_result


results['dt_down'] = dt_undersampled_result


# Create the pipeline with the best sampling approach (based on your results)
# Assuming oversampling performed best
dt_tuning_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('classifier', DecisionTreeClassifier(random_state=42))
])

# Define parameter grid
# ? NOTE: We add classifier__ as a prefix because we're using a pipeline in GridSearchCV.
param_grid = {
    'classifier__criterion': ['gini', 'entropy'],
    'classifier__max_depth': [None, 5, 10, 15, 20],  # None allows full growth
    'classifier__min_samples_split': [2, 5, 10, 20],  # Controls minimum samples for a split
    'classifier__min_samples_leaf': [1, 2, 5, 10],  # Controls minimum samples in a leaf,
    'classifier__max_features': [None, 'sqrt', 'log2'], # Controls max features to consider during split
    'classifier__class_weight': [None, 'balanced']
}

# Create stratified k-fold cross-validation
stratified_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Create grid search
grid_search = GridSearchCV(
    dt_tuning_pipeline,
    param_grid,
    scoring='f1',
    n_jobs=-1,
    verbose=1,
    cv = stratified_cv,
    #cv=5,
)

# Fit grid search
grid_search.fit(X_train, y_train)

Fitting 5 folds for each of 960 candidates, totalling 4800 fits

GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
             estimator=Pipeline(steps=[('preprocessor',
                                        ColumnTransformer(remainder='passthrough',
                                                          transformers=[('ordinal',
                                                                         OrdinalEncoder(categories=[['High '
                                                                                                     'School',
                                                                                                     "Bachelor's",
                                                                                                     "Master's",
                                                                                                     'Doctorate'],
                                                                                                    ['Micro '
                                                                                                     '(1-50)',
                                                                                                     'Small '
                                                                                                     '(51-250)',
                                                                                                     'Medium '
                                                                                                     '(251-1000)',
                                                                                                     'Large '
                                                                                                     '(1001-10000)',
                                                                                                     'Enterpris...
                                        DecisionTreeClassifier(random_state=42))]),
             n_jobs=-1,
             param_grid={'classifier__class_weight': [None, 'balanced'],
                         'classifier__criterion': ['gini', 'entropy'],
                         'classifier__max_depth': [None, 5, 10, 15, 20],
                         'classifier__max_features': [None, 'sqrt', 'log2'],
                         'classifier__min_samples_leaf': [1, 2, 5, 10],
                         'classifier__min_samples_split': [2, 5, 10, 20]},
             scoring='f1', verbose=1)

GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
             estimator=Pipeline(steps=[('preprocessor',
                                        ColumnTransformer(remainder='passthrough',
                                                          transformers=[('ordinal',
                                                                         OrdinalEncoder(categories=[['High '
                                                                                                     'School',
                                                                                                     "Bachelor's",
                                                                                                     "Master's",
                                                                                                     'Doctorate'],
                                                                                                    ['Micro '
                                                                                                     '(1-50)',
                                                                                                     'Small '
                                                                                                     '(51-250)',
                                                                                                     'Medium '
                                                                                                     '(251-1000)',
                                                                                                     'Large '
                                                                                                     '(1001-10000)',
                                                                                                     'Enterpris...
                                        DecisionTreeClassifier(random_state=42))]),
             n_jobs=-1,
             param_grid={'classifier__class_weight': [None, 'balanced'],
                         'classifier__criterion': ['gini', 'entropy'],
                         'classifier__max_depth': [None, 5, 10, 15, 20],
                         'classifier__max_features': [None, 'sqrt', 'log2'],
                         'classifier__min_samples_leaf': [1, 2, 5, 10],
                         'classifier__min_samples_split': [2, 5, 10, 20]},
             scoring='f1', verbose=1)

Pipeline(steps=[('preprocessor',
                 ColumnTransformer(remainder='passthrough',
                                   transformers=[('ordinal',
                                                  OrdinalEncoder(categories=[['High '
                                                                              'School',
                                                                              "Bachelor's",
                                                                              "Master's",
                                                                              'Doctorate'],
                                                                             ['Micro '
                                                                              '(1-50)',
                                                                              'Small '
                                                                              '(51-250)',
                                                                              'Medium '
                                                                              '(251-1000)',
                                                                              'Large '
                                                                              '(1001-10000)',
                                                                              'Enterprise '
                                                                              '(>10000)'],
                                                                             ['Legacy '
                                                                              '(Before '
                                                                              '1950)',
                                                                              'Traditional '
                                                                              'Era '
                                                                              '(1950-1979)',
                                                                              'Growth '
                                                                              'Ph...
                                                   'company_size',
                                                   'company_era',
                                                   'wage_category']),
                                                 ('binary',
                                                  OrdinalEncoder(categories=[['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y']]),
                                                  ['has_job_experience',
                                                   'requires_job_training',
                                                   'full_time_position']),
                                                 ('onehot',
                                                  OneHotEncoder(drop='first',
                                                                sparse_output=False),
                                                  ['continent',
                                                   'region_of_employment',
                                                   'unit_of_wage'])])),
                ('classifier', DecisionTreeClassifier(random_state=42))])

ColumnTransformer(remainder='passthrough',
                  transformers=[('ordinal',
                                 OrdinalEncoder(categories=[['High School',
                                                             "Bachelor's",
                                                             "Master's",
                                                             'Doctorate'],
                                                            ['Micro (1-50)',
                                                             'Small (51-250)',
                                                             'Medium '
                                                             '(251-1000)',
                                                             'Large '
                                                             '(1001-10000)',
                                                             'Enterprise '
                                                             '(>10000)'],
                                                            ['Legacy (Before '
                                                             '1950)',
                                                             'Traditional Era '
                                                             '(1950-1979)',
                                                             'Growth Phase '
                                                             '(1980-1999)',
                                                             'Digital '
                                                             'Transition Era '
                                                             '(...
                                                             '($80K-150K)',
                                                             'Executive '
                                                             '(>$150K)']]),
                                 ['education_of_employee', 'company_size',
                                  'company_era', 'wage_category']),
                                ('binary',
                                 OrdinalEncoder(categories=[['N', 'Y'],
                                                            ['N', 'Y'],
                                                            ['N', 'Y']]),
                                 ['has_job_experience', 'requires_job_training',
                                  'full_time_position']),
                                ('onehot',
                                 OneHotEncoder(drop='first',
                                               sparse_output=False),
                                 ['continent', 'region_of_employment',
                                  'unit_of_wage'])])

['education_of_employee', 'company_size', 'company_era', 'wage_category']

OrdinalEncoder(categories=[['High School', "Bachelor's", "Master's",
                            'Doctorate'],
                           ['Micro (1-50)', 'Small (51-250)',
                            'Medium (251-1000)', 'Large (1001-10000)',
                            'Enterprise (>10000)'],
                           ['Legacy (Before 1950)',
                            'Traditional Era (1950-1979)',
                            'Growth Phase (1980-1999)',
                            'Digital Transition Era (2000-2010)',
                            'AI & Mobile Era (2011-2016)'],
                           ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)',
                            'Professional ($80K-150K)', 'Executive (>$150K)']])

['has_job_experience', 'requires_job_training', 'full_time_position']

OrdinalEncoder(categories=[['N', 'Y'], ['N', 'Y'], ['N', 'Y']])

['continent', 'region_of_employment', 'unit_of_wage']


# Get best parameters
best_params = grid_search.best_params_
print(f"Best parameters: {best_params}")

Best parameters: {'classifier__class_weight': None, 'classifier__criterion': 'gini', 'classifier__max_depth': 5, 'classifier__max_features': None, 'classifier__min_samples_leaf': 5, 'classifier__min_samples_split': 20}


# Create tuned model with best parameters
dt_tuned_pipeline = ImbPipeline(
    [
        ("preprocessor", preprocessor),
        (
            "classifier",
            DecisionTreeClassifier(
                random_state=42,
                **{remove_pipeline_step_prefix(k): v for k, v in best_params.items()},
            ),
        ),
    ]
)

# Fit tuned model
dt_tuned_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = dt_tuned_pipeline.predict(X_test)


dt_tuned_result = get_classification_metrics(y_test, y_pred)
dt_tuned_result


results['dt_tuned'] = dt_tuned_result


model = dt_tuned_pipeline.named_steps['classifier']
feat_importances = get_feature_importances(model, transformed_columns)
feat_importances


plot_feature_importance(feat_importances, top_n=10)


# Create pipeline
bagging_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', BaggingClassifier(
        random_state=42,
        n_estimators=100,  # Number of base estimators
        max_samples=0.8,   # Each estimator gets 80% of samples
        max_features=0.8,  # Each estimator gets 80% of features
        bootstrap=True,    # Sample with replacement
        bootstrap_features=True,  # Sample features with replacement
        n_jobs=-1          # Use all available cores
    ))
])

# Fit the pipeline on training data
bagging_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = bagging_pipeline.predict(X_test)


bagging_result = get_classification_metrics(y_test, y_pred)
bagging_result


results['bc'] = bagging_result


# Create pipeline
bagging_oversampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', oversampler),
    ('classifier', BaggingClassifier(
        random_state=42,
        n_estimators=100,
        max_samples=0.8,
        max_features=0.8,
        bootstrap=True,
        bootstrap_features=True,
        n_jobs=-1
    ))
])

# Fit the pipeline on training data
bagging_oversampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = bagging_oversampled_pipeline.predict(X_test)


bagging_oversampled_result = get_classification_metrics(y_test, y_pred)
bagging_oversampled_result


results['bc_up'] = bagging_oversampled_result


# Create pipeline
bagging_undersampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', undersampler),
    ('classifier', BaggingClassifier(
        random_state=42,
        n_estimators=100,
        max_samples=0.8,
        max_features=0.8,
        bootstrap=True,
        bootstrap_features=True,
        n_jobs=-1
    ))
])

# Fit the pipeline on training data
bagging_undersampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = bagging_undersampled_pipeline.predict(X_test)


bagging_undersampled_result = get_classification_metrics(y_test, y_pred)
bagging_undersampled_result


results['bc_down'] = bagging_undersampled_result


# Create stratified k-fold cross-validation
stratified_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Create the pipeline
bagging_tuning_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', BaggingClassifier(random_state=42))
])

# Define hyperparameter grid
param_grid = {
    'classifier__n_estimators': [10, 50, 100, 200],
    'classifier__max_samples': [0.5, 0.7, 1.0],
    'classifier__max_features': [0.5, 0.7, 1.0],
    'classifier__bootstrap': [True, False],
    'classifier__bootstrap_features': [True, False]
}

# Create grid search with stratified CV
grid_search = GridSearchCV(
    bagging_tuning_pipeline,
    param_grid,
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1
)

# Fit grid search
grid_search.fit(X_train, y_train)

Fitting 5 folds for each of 144 candidates, totalling 720 fits

GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
             estimator=Pipeline(steps=[('preprocessor',
                                        ColumnTransformer(remainder='passthrough',
                                                          transformers=[('ordinal',
                                                                         OrdinalEncoder(categories=[['High '
                                                                                                     'School',
                                                                                                     "Bachelor's",
                                                                                                     "Master's",
                                                                                                     'Doctorate'],
                                                                                                    ['Micro '
                                                                                                     '(1-50)',
                                                                                                     'Small '
                                                                                                     '(51-250)',
                                                                                                     'Medium '
                                                                                                     '(251-1000)',
                                                                                                     'Large '
                                                                                                     '(1001-10000)',
                                                                                                     'Enterpris...
                                                                         ['continent',
                                                                          'region_of_employment',
                                                                          'unit_of_wage'])])),
                                       ('classifier',
                                        BaggingClassifier(random_state=42))]),
             n_jobs=-1,
             param_grid={'classifier__bootstrap': [True, False],
                         'classifier__bootstrap_features': [True, False],
                         'classifier__max_features': [0.5, 0.7, 1.0],
                         'classifier__max_samples': [0.5, 0.7, 1.0],
                         'classifier__n_estimators': [10, 50, 100, 200]},
             scoring='f1', verbose=1)

GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
             estimator=Pipeline(steps=[('preprocessor',
                                        ColumnTransformer(remainder='passthrough',
                                                          transformers=[('ordinal',
                                                                         OrdinalEncoder(categories=[['High '
                                                                                                     'School',
                                                                                                     "Bachelor's",
                                                                                                     "Master's",
                                                                                                     'Doctorate'],
                                                                                                    ['Micro '
                                                                                                     '(1-50)',
                                                                                                     'Small '
                                                                                                     '(51-250)',
                                                                                                     'Medium '
                                                                                                     '(251-1000)',
                                                                                                     'Large '
                                                                                                     '(1001-10000)',
                                                                                                     'Enterpris...
                                                                         ['continent',
                                                                          'region_of_employment',
                                                                          'unit_of_wage'])])),
                                       ('classifier',
                                        BaggingClassifier(random_state=42))]),
             n_jobs=-1,
             param_grid={'classifier__bootstrap': [True, False],
                         'classifier__bootstrap_features': [True, False],
                         'classifier__max_features': [0.5, 0.7, 1.0],
                         'classifier__max_samples': [0.5, 0.7, 1.0],
                         'classifier__n_estimators': [10, 50, 100, 200]},
             scoring='f1', verbose=1)

Pipeline(steps=[('preprocessor',
                 ColumnTransformer(remainder='passthrough',
                                   transformers=[('ordinal',
                                                  OrdinalEncoder(categories=[['High '
                                                                              'School',
                                                                              "Bachelor's",
                                                                              "Master's",
                                                                              'Doctorate'],
                                                                             ['Micro '
                                                                              '(1-50)',
                                                                              'Small '
                                                                              '(51-250)',
                                                                              'Medium '
                                                                              '(251-1000)',
                                                                              'Large '
                                                                              '(1001-10000)',
                                                                              'Enterprise '
                                                                              '(>10000)'],
                                                                             ['Legacy '
                                                                              '(Before '
                                                                              '1950)',
                                                                              'Traditional '
                                                                              'Era '
                                                                              '(1950-1979)',
                                                                              'Growth '
                                                                              'Ph...
                                                   'company_size',
                                                   'company_era',
                                                   'wage_category']),
                                                 ('binary',
                                                  OrdinalEncoder(categories=[['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y']]),
                                                  ['has_job_experience',
                                                   'requires_job_training',
                                                   'full_time_position']),
                                                 ('onehot',
                                                  OneHotEncoder(drop='first',
                                                                sparse_output=False),
                                                  ['continent',
                                                   'region_of_employment',
                                                   'unit_of_wage'])])),
                ('classifier', BaggingClassifier(random_state=42))])

ColumnTransformer(remainder='passthrough',
                  transformers=[('ordinal',
                                 OrdinalEncoder(categories=[['High School',
                                                             "Bachelor's",
                                                             "Master's",
                                                             'Doctorate'],
                                                            ['Micro (1-50)',
                                                             'Small (51-250)',
                                                             'Medium '
                                                             '(251-1000)',
                                                             'Large '
                                                             '(1001-10000)',
                                                             'Enterprise '
                                                             '(>10000)'],
                                                            ['Legacy (Before '
                                                             '1950)',
                                                             'Traditional Era '
                                                             '(1950-1979)',
                                                             'Growth Phase '
                                                             '(1980-1999)',
                                                             'Digital '
                                                             'Transition Era '
                                                             '(...
                                                             '($80K-150K)',
                                                             'Executive '
                                                             '(>$150K)']]),
                                 ['education_of_employee', 'company_size',
                                  'company_era', 'wage_category']),
                                ('binary',
                                 OrdinalEncoder(categories=[['N', 'Y'],
                                                            ['N', 'Y'],
                                                            ['N', 'Y']]),
                                 ['has_job_experience', 'requires_job_training',
                                  'full_time_position']),
                                ('onehot',
                                 OneHotEncoder(drop='first',
                                               sparse_output=False),
                                 ['continent', 'region_of_employment',
                                  'unit_of_wage'])])

['education_of_employee', 'company_size', 'company_era', 'wage_category']

OrdinalEncoder(categories=[['High School', "Bachelor's", "Master's",
                            'Doctorate'],
                           ['Micro (1-50)', 'Small (51-250)',
                            'Medium (251-1000)', 'Large (1001-10000)',
                            'Enterprise (>10000)'],
                           ['Legacy (Before 1950)',
                            'Traditional Era (1950-1979)',
                            'Growth Phase (1980-1999)',
                            'Digital Transition Era (2000-2010)',
                            'AI & Mobile Era (2011-2016)'],
                           ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)',
                            'Professional ($80K-150K)', 'Executive (>$150K)']])

['has_job_experience', 'requires_job_training', 'full_time_position']

OrdinalEncoder(categories=[['N', 'Y'], ['N', 'Y'], ['N', 'Y']])

['continent', 'region_of_employment', 'unit_of_wage']


# Get best parameters
best_params = grid_search.best_params_
print(f"Best parameters: {best_params}")

Best parameters: {'classifier__bootstrap': True, 'classifier__bootstrap_features': False, 'classifier__max_features': 0.7, 'classifier__max_samples': 0.5, 'classifier__n_estimators': 200}


# Create tuned model with best parameters
bagging_tuned_pipeline = Pipeline(
    [
        ("preprocessor", preprocessor),
        (
            "classifier",
            BaggingClassifier(
                random_state=42,
                **{remove_pipeline_step_prefix(k): v for k, v in best_params.items()},
            ),
        ),
    ]
)

# Fit tuned model
bagging_tuned_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = bagging_tuned_pipeline.predict(X_test)


bagging_tuned_result = get_classification_metrics(y_test, y_pred)
bagging_tuned_result


results['bc_tuned'] = bagging_tuned_result


# Create pipeline
rf_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(
        random_state=42,
        n_jobs=-1,           # Use all available cores
        class_weight='balanced'
    ))
])

# Fit the pipeline on training data
rf_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = rf_pipeline.predict(X_test)


rf_result = get_classification_metrics(y_test, y_pred)
rf_result


results['rf'] = rf_result


# Create pipeline
rf_oversampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', oversampler),
    ('classifier', RandomForestClassifier(
        random_state=42,
        n_jobs=-1
    ))
])

# Fit the pipeline on training data
rf_oversampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = rf_oversampled_pipeline.predict(X_test)


rf_oversampled_result = get_classification_metrics(y_test, y_pred)
rf_oversampled_result


results['rf_up'] = rf_oversampled_result


# Create pipeline with preprocessing, undersampling, and random forest classifier
rf_undersampled_pipeline = ImbPipeline(
    [
        ("preprocessor", preprocessor),
        ("sampler", undersampler),
        (
            "classifier",
            RandomForestClassifier(
                random_state=42,
                n_jobs=-1,
            ),
        ),
    ]
)

# Fit the pipeline on training data
rf_undersampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = rf_undersampled_pipeline.predict(X_test)


rf_undersampled_result = get_classification_metrics(y_test, y_pred)
rf_undersampled_result


results['rf_down'] = rf_undersampled_result


# Create stratified k-fold cross-validation
stratified_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Create the pipeline (using normal approach)
rf_tuning_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(random_state=42))
])

# Define parameter distributions
param_distributions = {
    'classifier__n_estimators': randint(50, 500),
    'classifier__max_depth': [None] + list(randint(5, 50).rvs(5)),
    'classifier__min_samples_split': randint(2, 20),
    'classifier__min_samples_leaf': randint(1, 10),
    'classifier__max_features': ['sqrt', 'log2', None],
    'classifier__bootstrap': [True, False],
    'classifier__class_weight': [None, 'balanced', 'balanced_subsample'],
    'classifier__criterion': ['gini', 'entropy']
}

# Create randomized search with stratified CV
random_search = RandomizedSearchCV(
    rf_tuning_pipeline,
    param_distributions=param_distributions,
    n_iter=50,  # Number of parameter settings sampled
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1,
    random_state=42
)

# Fit grid search
random_search.fit(X_train, y_train)

Fitting 5 folds for each of 50 candidates, totalling 250 fits

RandomizedSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
                   estimator=Pipeline(steps=[('preprocessor',
                                              ColumnTransformer(remainder='passthrough',
                                                                transformers=[('ordinal',
                                                                               OrdinalEncoder(categories=[['High '
                                                                                                           'School',
                                                                                                           "Bachelor's",
                                                                                                           "Master's",
                                                                                                           'Doctorate'],
                                                                                                          ['Micro '
                                                                                                           '(1-50)',
                                                                                                           'Small '
                                                                                                           '(51-250)',
                                                                                                           'Medium '
                                                                                                           '(251-1000)',
                                                                                                           'Large '
                                                                                                           '(1001-10000)',
                                                                                                           'Ent...
                                        'classifier__min_samples_leaf': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307e49ed0>,
                                        'classifier__min_samples_split': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307ffc550>,
                                        'classifier__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307f371d0>},
                   random_state=42, scoring='f1', verbose=1)

RandomizedSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
                   estimator=Pipeline(steps=[('preprocessor',
                                              ColumnTransformer(remainder='passthrough',
                                                                transformers=[('ordinal',
                                                                               OrdinalEncoder(categories=[['High '
                                                                                                           'School',
                                                                                                           "Bachelor's",
                                                                                                           "Master's",
                                                                                                           'Doctorate'],
                                                                                                          ['Micro '
                                                                                                           '(1-50)',
                                                                                                           'Small '
                                                                                                           '(51-250)',
                                                                                                           'Medium '
                                                                                                           '(251-1000)',
                                                                                                           'Large '
                                                                                                           '(1001-10000)',
                                                                                                           'Ent...
                                        'classifier__min_samples_leaf': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307e49ed0>,
                                        'classifier__min_samples_split': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307ffc550>,
                                        'classifier__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307f371d0>},
                   random_state=42, scoring='f1', verbose=1)

Pipeline(steps=[('preprocessor',
                 ColumnTransformer(remainder='passthrough',
                                   transformers=[('ordinal',
                                                  OrdinalEncoder(categories=[['High '
                                                                              'School',
                                                                              "Bachelor's",
                                                                              "Master's",
                                                                              'Doctorate'],
                                                                             ['Micro '
                                                                              '(1-50)',
                                                                              'Small '
                                                                              '(51-250)',
                                                                              'Medium '
                                                                              '(251-1000)',
                                                                              'Large '
                                                                              '(1001-10000)',
                                                                              'Enterprise '
                                                                              '(>10000)'],
                                                                             ['Legacy '
                                                                              '(Before '
                                                                              '1950)',
                                                                              'Traditional '
                                                                              'Era '
                                                                              '(1950-1979)',
                                                                              'Growth '
                                                                              'Ph...
                                                   'company_size',
                                                   'company_era',
                                                   'wage_category']),
                                                 ('binary',
                                                  OrdinalEncoder(categories=[['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y']]),
                                                  ['has_job_experience',
                                                   'requires_job_training',
                                                   'full_time_position']),
                                                 ('onehot',
                                                  OneHotEncoder(drop='first',
                                                                sparse_output=False),
                                                  ['continent',
                                                   'region_of_employment',
                                                   'unit_of_wage'])])),
                ('classifier', RandomForestClassifier(random_state=42))])

ColumnTransformer(remainder='passthrough',
                  transformers=[('ordinal',
                                 OrdinalEncoder(categories=[['High School',
                                                             "Bachelor's",
                                                             "Master's",
                                                             'Doctorate'],
                                                            ['Micro (1-50)',
                                                             'Small (51-250)',
                                                             'Medium '
                                                             '(251-1000)',
                                                             'Large '
                                                             '(1001-10000)',
                                                             'Enterprise '
                                                             '(>10000)'],
                                                            ['Legacy (Before '
                                                             '1950)',
                                                             'Traditional Era '
                                                             '(1950-1979)',
                                                             'Growth Phase '
                                                             '(1980-1999)',
                                                             'Digital '
                                                             'Transition Era '
                                                             '(...
                                                             '($80K-150K)',
                                                             'Executive '
                                                             '(>$150K)']]),
                                 ['education_of_employee', 'company_size',
                                  'company_era', 'wage_category']),
                                ('binary',
                                 OrdinalEncoder(categories=[['N', 'Y'],
                                                            ['N', 'Y'],
                                                            ['N', 'Y']]),
                                 ['has_job_experience', 'requires_job_training',
                                  'full_time_position']),
                                ('onehot',
                                 OneHotEncoder(drop='first',
                                               sparse_output=False),
                                 ['continent', 'region_of_employment',
                                  'unit_of_wage'])])

['education_of_employee', 'company_size', 'company_era', 'wage_category']

OrdinalEncoder(categories=[['High School', "Bachelor's", "Master's",
                            'Doctorate'],
                           ['Micro (1-50)', 'Small (51-250)',
                            'Medium (251-1000)', 'Large (1001-10000)',
                            'Enterprise (>10000)'],
                           ['Legacy (Before 1950)',
                            'Traditional Era (1950-1979)',
                            'Growth Phase (1980-1999)',
                            'Digital Transition Era (2000-2010)',
                            'AI & Mobile Era (2011-2016)'],
                           ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)',
                            'Professional ($80K-150K)', 'Executive (>$150K)']])

['has_job_experience', 'requires_job_training', 'full_time_position']

OrdinalEncoder(categories=[['N', 'Y'], ['N', 'Y'], ['N', 'Y']])

['continent', 'region_of_employment', 'unit_of_wage']


# Get best parameters
best_params = random_search.best_params_
print(f"Best parameters: {best_params}")

Best parameters: {'classifier__bootstrap': True, 'classifier__class_weight': None, 'classifier__criterion': 'entropy', 'classifier__max_depth': 36, 'classifier__max_features': 'sqrt', 'classifier__min_samples_leaf': 9, 'classifier__min_samples_split': 19, 'classifier__n_estimators': 200}


# Create tuned model with best parameters
rf_tuned_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(random_state=42, **{remove_pipeline_step_prefix(k): v for k, v in best_params.items()}))
])

# Fit tuned model
rf_tuned_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = rf_tuned_pipeline.predict(X_test)


rf_tuned_result = get_classification_metrics(y_test, y_pred)
rf_tuned_result


results['rf_tuned'] = rf_tuned_result


# Feature Importance
model = rf_tuned_pipeline.named_steps['classifier']
feat_importances = get_feature_importances(model, transformed_columns)
feat_importances


plot_feature_importance(feat_importances, top_n=10)


# Create pipeline
ada_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', AdaBoostClassifier(
        base_estimator=DecisionTreeClassifier(max_depth=3),  # Using shallow trees as base estimators
        n_estimators=100,  # Number of estimators
        random_state=42
    ))
])

# Fit the pipeline on training data
ada_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = ada_pipeline.predict(X_test)


ada_result = get_classification_metrics(y_test, y_pred)
ada_result


results['ada'] = ada_result


# Create pipeline
ada_oversampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', oversampler),
    ('classifier', AdaBoostClassifier(
        base_estimator=DecisionTreeClassifier(max_depth=3),
        n_estimators=100,
        learning_rate=1.0,
        random_state=42
    ))
])

# Fit the pipeline on training data
ada_oversampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = ada_oversampled_pipeline.predict(X_test)


ada_oversampled_result = get_classification_metrics(y_test, y_pred)
ada_oversampled_result


results['ada_up'] = ada_oversampled_result


# Create pipeline
ada_undersampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', undersampler),
    ('classifier', AdaBoostClassifier(
        base_estimator=DecisionTreeClassifier(max_depth=3),
        n_estimators=100,
        random_state=42
    ))
])

# Fit the pipeline on training data
ada_undersampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = ada_undersampled_pipeline.predict(X_test)


ada_undersampled_result = get_classification_metrics(y_test, y_pred)
ada_undersampled_result


results['ada_down'] = ada_undersampled_result


# Create stratified k-fold cross-validation
stratified_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Create the pipeline (using normal approach)
ada_tuning_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', AdaBoostClassifier(
        base_estimator=DecisionTreeClassifier(max_depth=1), # Decision Stump
        random_state=42,
    ))
])

# Define parameter distributions
param_distributions = {
    'classifier__n_estimators': randint(50, 300),
    'classifier__learning_rate': uniform(0.01, 0.99),  # From 0.01 to 1.0
    'classifier__base_estimator__max_depth': randint(1, 4),
    'classifier__base_estimator__min_samples_split': randint(2, 20),
    'classifier__base_estimator__min_samples_leaf': randint(1, 10),
    'classifier__algorithm': ['SAMME', 'SAMME.R']
}

# Create randomized search with stratified CV
random_search = RandomizedSearchCV(
    ada_tuning_pipeline,
    param_distributions=param_distributions,
    n_iter=50,
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1,
    random_state=42
)

# Fit randomized search
random_search.fit(X_train, y_train)

Fitting 5 folds for each of 50 candidates, totalling 250 fits

/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(
/Users/nipunshah/anaconda3/lib/python3.11/site-packages/sklearn/ensemble/_base.py:166: FutureWarning: `base_estimator` was renamed to `estimator` in version 1.2 and will be removed in 1.4.
  warnings.warn(

RandomizedSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
                   estimator=Pipeline(steps=[('preprocessor',
                                              ColumnTransformer(remainder='passthrough',
                                                                transformers=[('ordinal',
                                                                               OrdinalEncoder(categories=[['High '
                                                                                                           'School',
                                                                                                           "Bachelor's",
                                                                                                           "Master's",
                                                                                                           'Doctorate'],
                                                                                                          ['Micro '
                                                                                                           '(1-50)',
                                                                                                           'Small '
                                                                                                           '(51-250)',
                                                                                                           'Medium '
                                                                                                           '(251-1000)',
                                                                                                           'Large '
                                                                                                           '(1001-10000)',
                                                                                                           'Ent...
                                        'classifier__base_estimator__min_samples_split': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307f69e10>,
                                        'classifier__learning_rate': <scipy.stats._distn_infrastructure.rv_continuous_frozen object at 0x307f6be90>,
                                        'classifier__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307f69590>},
                   random_state=42, scoring='f1', verbose=1)

RandomizedSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
                   estimator=Pipeline(steps=[('preprocessor',
                                              ColumnTransformer(remainder='passthrough',
                                                                transformers=[('ordinal',
                                                                               OrdinalEncoder(categories=[['High '
                                                                                                           'School',
                                                                                                           "Bachelor's",
                                                                                                           "Master's",
                                                                                                           'Doctorate'],
                                                                                                          ['Micro '
                                                                                                           '(1-50)',
                                                                                                           'Small '
                                                                                                           '(51-250)',
                                                                                                           'Medium '
                                                                                                           '(251-1000)',
                                                                                                           'Large '
                                                                                                           '(1001-10000)',
                                                                                                           'Ent...
                                        'classifier__base_estimator__min_samples_split': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307f69e10>,
                                        'classifier__learning_rate': <scipy.stats._distn_infrastructure.rv_continuous_frozen object at 0x307f6be90>,
                                        'classifier__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307f69590>},
                   random_state=42, scoring='f1', verbose=1)

Pipeline(steps=[('preprocessor',
                 ColumnTransformer(remainder='passthrough',
                                   transformers=[('ordinal',
                                                  OrdinalEncoder(categories=[['High '
                                                                              'School',
                                                                              "Bachelor's",
                                                                              "Master's",
                                                                              'Doctorate'],
                                                                             ['Micro '
                                                                              '(1-50)',
                                                                              'Small '
                                                                              '(51-250)',
                                                                              'Medium '
                                                                              '(251-1000)',
                                                                              'Large '
                                                                              '(1001-10000)',
                                                                              'Enterprise '
                                                                              '(>10000)'],
                                                                             ['Legacy '
                                                                              '(Before '
                                                                              '1950)',
                                                                              'Traditional '
                                                                              'Era '
                                                                              '(1950-1979)',
                                                                              'Growth '
                                                                              'Ph...
                                                 ('binary',
                                                  OrdinalEncoder(categories=[['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y']]),
                                                  ['has_job_experience',
                                                   'requires_job_training',
                                                   'full_time_position']),
                                                 ('onehot',
                                                  OneHotEncoder(drop='first',
                                                                sparse_output=False),
                                                  ['continent',
                                                   'region_of_employment',
                                                   'unit_of_wage'])])),
                ('classifier',
                 AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=1),
                                    random_state=42))])

ColumnTransformer(remainder='passthrough',
                  transformers=[('ordinal',
                                 OrdinalEncoder(categories=[['High School',
                                                             "Bachelor's",
                                                             "Master's",
                                                             'Doctorate'],
                                                            ['Micro (1-50)',
                                                             'Small (51-250)',
                                                             'Medium '
                                                             '(251-1000)',
                                                             'Large '
                                                             '(1001-10000)',
                                                             'Enterprise '
                                                             '(>10000)'],
                                                            ['Legacy (Before '
                                                             '1950)',
                                                             'Traditional Era '
                                                             '(1950-1979)',
                                                             'Growth Phase '
                                                             '(1980-1999)',
                                                             'Digital '
                                                             'Transition Era '
                                                             '(...
                                                             '($80K-150K)',
                                                             'Executive '
                                                             '(>$150K)']]),
                                 ['education_of_employee', 'company_size',
                                  'company_era', 'wage_category']),
                                ('binary',
                                 OrdinalEncoder(categories=[['N', 'Y'],
                                                            ['N', 'Y'],
                                                            ['N', 'Y']]),
                                 ['has_job_experience', 'requires_job_training',
                                  'full_time_position']),
                                ('onehot',
                                 OneHotEncoder(drop='first',
                                               sparse_output=False),
                                 ['continent', 'region_of_employment',
                                  'unit_of_wage'])])

['education_of_employee', 'company_size', 'company_era', 'wage_category']

OrdinalEncoder(categories=[['High School', "Bachelor's", "Master's",
                            'Doctorate'],
                           ['Micro (1-50)', 'Small (51-250)',
                            'Medium (251-1000)', 'Large (1001-10000)',
                            'Enterprise (>10000)'],
                           ['Legacy (Before 1950)',
                            'Traditional Era (1950-1979)',
                            'Growth Phase (1980-1999)',
                            'Digital Transition Era (2000-2010)',
                            'AI & Mobile Era (2011-2016)'],
                           ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)',
                            'Professional ($80K-150K)', 'Executive (>$150K)']])

['has_job_experience', 'requires_job_training', 'full_time_position']

OrdinalEncoder(categories=[['N', 'Y'], ['N', 'Y'], ['N', 'Y']])


# Get best parameters
best_params = random_search.best_params_
print(f"Best parameters: {best_params}")

Best parameters: {'classifier__algorithm': 'SAMME', 'classifier__base_estimator__max_depth': 3, 'classifier__base_estimator__min_samples_leaf': 9, 'classifier__base_estimator__min_samples_split': 14, 'classifier__learning_rate': 0.86212327742378, 'classifier__n_estimators': 152}


# Create tuned model with best parameters
# Need to handle the base_estimator parameters separately
base_estimator_params = {k.replace('classifier__base_estimator__', ''): v
                         for k, v in best_params.items()
                         if k.startswith('classifier__base_estimator__')}

classifier_params = {k.replace('classifier__', ''): v
                     for k, v in best_params.items()
                     if not k.startswith('classifier__base_estimator__')}

# Remove base_estimator from classifier_params if it exists
if 'base_estimator' in classifier_params:
    del classifier_params['base_estimator']

# Create the tuned model
ada_tuned_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', AdaBoostClassifier(
        base_estimator=DecisionTreeClassifier(**base_estimator_params),
        random_state=42,
        **classifier_params
    ))
])

# Fit tuned model
ada_tuned_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = ada_tuned_pipeline.predict(X_test)


ada_tuned_result = get_classification_metrics(y_test, y_pred)
ada_tuned_result


results['ada_tuned'] = ada_tuned_result


# Feature Importance
model = ada_tuned_pipeline.named_steps['classifier']
feat_importances = get_feature_importances(model, transformed_columns)
feat_importances


plot_feature_importance(feat_importances, top_n=10)


# Create pipeline with preprocessing and Gradient Boosting classifier
gb_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', GradientBoostingClassifier(
        n_estimators=100,
        learning_rate=0.1,
        max_depth=3,
        subsample=0.8,
        random_state=42
    ))
])

# Fit the pipeline on training data
gb_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = gb_pipeline.predict(X_test)


gb_result = get_classification_metrics(y_test, y_pred)
gb_result


results['gb'] = gb_result


# Create pipeline with preprocessing, oversampling, and Gradient Boosting classifier
gb_oversampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', oversampler),
    ('classifier', GradientBoostingClassifier(
        n_estimators=100,
        learning_rate=0.1,
        max_depth=3,
        subsample=0.8,
        random_state=42
    ))
])

# Fit the pipeline on training data
gb_oversampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = gb_oversampled_pipeline.predict(X_test)


gb_oversampled_result = get_classification_metrics(y_test, y_pred)
gb_oversampled_result


results['gb_up'] = gb_oversampled_result


# Create pipeline
gb_undersampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', undersampler),
    ('classifier', GradientBoostingClassifier(
        n_estimators=100,
        learning_rate=0.1,
        max_depth=3,
        subsample=0.8,
        random_state=42
    ))
])

# Fit the pipeline on training data
gb_undersampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = gb_undersampled_pipeline.predict(X_test)


gb_undersampled_result = get_classification_metrics(y_test, y_pred)
gb_undersampled_result


results['gb_down'] = gb_undersampled_result


# Create stratified k-fold cross-validation
stratified_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Create the pipeline (using normal approach since it performed well)
gb_tuning_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', GradientBoostingClassifier(random_state=42))
])

# Define parameter distributions
param_distributions = {
    'classifier__n_estimators': randint(50, 500),
    'classifier__learning_rate': uniform(0.01, 0.3),
    'classifier__max_depth': randint(3, 8),
    'classifier__min_samples_split': randint(2, 15),
    'classifier__min_samples_leaf': randint(1, 10),
    'classifier__subsample': uniform(0.7, 0.3),  # From 0.7 to 1.0
    'classifier__max_features': ['sqrt', 'log2', None]
}

# Create randomized search with stratified CV
random_search = RandomizedSearchCV(
    gb_tuning_pipeline,
    param_distributions=param_distributions,
    n_iter=50,  # 50 random samples
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1,
    random_state=42
)

# Fit randomized search
random_search.fit(X_train, y_train)

Fitting 5 folds for each of 50 candidates, totalling 250 fits

RandomizedSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
                   estimator=Pipeline(steps=[('preprocessor',
                                              ColumnTransformer(remainder='passthrough',
                                                                transformers=[('ordinal',
                                                                               OrdinalEncoder(categories=[['High '
                                                                                                           'School',
                                                                                                           "Bachelor's",
                                                                                                           "Master's",
                                                                                                           'Doctorate'],
                                                                                                          ['Micro '
                                                                                                           '(1-50)',
                                                                                                           'Small '
                                                                                                           '(51-250)',
                                                                                                           'Medium '
                                                                                                           '(251-1000)',
                                                                                                           'Large '
                                                                                                           '(1001-10000)',
                                                                                                           'Ent...
                                        'classifier__min_samples_leaf': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307ba6510>,
                                        'classifier__min_samples_split': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307ba6f10>,
                                        'classifier__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307fdcad0>,
                                        'classifier__subsample': <scipy.stats._distn_infrastructure.rv_continuous_frozen object at 0x307ba5b50>},
                   random_state=42, scoring='f1', verbose=1)

RandomizedSearchCV(cv=StratifiedKFold(n_splits=5, random_state=42, shuffle=True),
                   estimator=Pipeline(steps=[('preprocessor',
                                              ColumnTransformer(remainder='passthrough',
                                                                transformers=[('ordinal',
                                                                               OrdinalEncoder(categories=[['High '
                                                                                                           'School',
                                                                                                           "Bachelor's",
                                                                                                           "Master's",
                                                                                                           'Doctorate'],
                                                                                                          ['Micro '
                                                                                                           '(1-50)',
                                                                                                           'Small '
                                                                                                           '(51-250)',
                                                                                                           'Medium '
                                                                                                           '(251-1000)',
                                                                                                           'Large '
                                                                                                           '(1001-10000)',
                                                                                                           'Ent...
                                        'classifier__min_samples_leaf': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307ba6510>,
                                        'classifier__min_samples_split': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307ba6f10>,
                                        'classifier__n_estimators': <scipy.stats._distn_infrastructure.rv_discrete_frozen object at 0x307fdcad0>,
                                        'classifier__subsample': <scipy.stats._distn_infrastructure.rv_continuous_frozen object at 0x307ba5b50>},
                   random_state=42, scoring='f1', verbose=1)

Pipeline(steps=[('preprocessor',
                 ColumnTransformer(remainder='passthrough',
                                   transformers=[('ordinal',
                                                  OrdinalEncoder(categories=[['High '
                                                                              'School',
                                                                              "Bachelor's",
                                                                              "Master's",
                                                                              'Doctorate'],
                                                                             ['Micro '
                                                                              '(1-50)',
                                                                              'Small '
                                                                              '(51-250)',
                                                                              'Medium '
                                                                              '(251-1000)',
                                                                              'Large '
                                                                              '(1001-10000)',
                                                                              'Enterprise '
                                                                              '(>10000)'],
                                                                             ['Legacy '
                                                                              '(Before '
                                                                              '1950)',
                                                                              'Traditional '
                                                                              'Era '
                                                                              '(1950-1979)',
                                                                              'Growth '
                                                                              'Ph...
                                                   'company_size',
                                                   'company_era',
                                                   'wage_category']),
                                                 ('binary',
                                                  OrdinalEncoder(categories=[['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y'],
                                                                             ['N',
                                                                              'Y']]),
                                                  ['has_job_experience',
                                                   'requires_job_training',
                                                   'full_time_position']),
                                                 ('onehot',
                                                  OneHotEncoder(drop='first',
                                                                sparse_output=False),
                                                  ['continent',
                                                   'region_of_employment',
                                                   'unit_of_wage'])])),
                ('classifier', GradientBoostingClassifier(random_state=42))])

ColumnTransformer(remainder='passthrough',
                  transformers=[('ordinal',
                                 OrdinalEncoder(categories=[['High School',
                                                             "Bachelor's",
                                                             "Master's",
                                                             'Doctorate'],
                                                            ['Micro (1-50)',
                                                             'Small (51-250)',
                                                             'Medium '
                                                             '(251-1000)',
                                                             'Large '
                                                             '(1001-10000)',
                                                             'Enterprise '
                                                             '(>10000)'],
                                                            ['Legacy (Before '
                                                             '1950)',
                                                             'Traditional Era '
                                                             '(1950-1979)',
                                                             'Growth Phase '
                                                             '(1980-1999)',
                                                             'Digital '
                                                             'Transition Era '
                                                             '(...
                                                             '($80K-150K)',
                                                             'Executive '
                                                             '(>$150K)']]),
                                 ['education_of_employee', 'company_size',
                                  'company_era', 'wage_category']),
                                ('binary',
                                 OrdinalEncoder(categories=[['N', 'Y'],
                                                            ['N', 'Y'],
                                                            ['N', 'Y']]),
                                 ['has_job_experience', 'requires_job_training',
                                  'full_time_position']),
                                ('onehot',
                                 OneHotEncoder(drop='first',
                                               sparse_output=False),
                                 ['continent', 'region_of_employment',
                                  'unit_of_wage'])])

['education_of_employee', 'company_size', 'company_era', 'wage_category']

OrdinalEncoder(categories=[['High School', "Bachelor's", "Master's",
                            'Doctorate'],
                           ['Micro (1-50)', 'Small (51-250)',
                            'Medium (251-1000)', 'Large (1001-10000)',
                            'Enterprise (>10000)'],
                           ['Legacy (Before 1950)',
                            'Traditional Era (1950-1979)',
                            'Growth Phase (1980-1999)',
                            'Digital Transition Era (2000-2010)',
                            'AI & Mobile Era (2011-2016)'],
                           ['Entry-level ($0-40K)', 'Mid-level ($40K-80K)',
                            'Professional ($80K-150K)', 'Executive (>$150K)']])

['has_job_experience', 'requires_job_training', 'full_time_position']

OrdinalEncoder(categories=[['N', 'Y'], ['N', 'Y'], ['N', 'Y']])

['continent', 'region_of_employment', 'unit_of_wage']


# Get best parameters
best_params = random_search.best_params_
print(f"Best parameters: {best_params}")

Best parameters: {'classifier__learning_rate': 0.022130076861529402, 'classifier__max_depth': 4, 'classifier__max_features': 'log2', 'classifier__min_samples_leaf': 3, 'classifier__min_samples_split': 8, 'classifier__n_estimators': 290, 'classifier__subsample': 0.794306794322898}


# Create tuned model with best parameters
gb_tuned_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', GradientBoostingClassifier(random_state=42, **{remove_pipeline_step_prefix(k): v for k, v in best_params.items()}))
])

# Fit tuned model
gb_tuned_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = gb_tuned_pipeline.predict(X_test)


gb_tuned_result = get_classification_metrics(y_test, y_pred)
gb_tuned_result


results['gb_tuned'] = gb_tuned_result


# Feature Importance
model = gb_tuned_pipeline.named_steps['classifier']
feat_importances = get_feature_importances(model, transformed_columns)
feat_importances


plot_feature_importance(feat_importances, top_n=10)


# Create pipeline
xgb_pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', XGBClassifier(
        n_estimators=100,   # Boosting rounds
        learning_rate=0.1,  # Standard learning rate
        max_depth=5,        # Moderate depth to balance learning capacity
        subsample=0.8,      # Randomly sample 80% of data per tree
        min_child_weight=1,   # default; allows splitting with minimal constraints
        colsample_bytree=0.8,  # Use 80% of features for each tree
        objective='binary:logistic',
        eval_metric='logloss',  # Standard for binary classification
        gamma=0,                # No split regularization initially
        scale_pos_weight=0.5,   # Adjust for 66%/33% class imbalance
        use_label_encoder=False,  # Prevents warnings
        random_state=42,
    ))
])

# Fit the pipeline on training data
xgb_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = xgb_pipeline.predict(X_test)


xgb_result = get_classification_metrics(y_test, y_pred)
xgb_result


results['xgb'] = xgb_result


# Create pipeline with preprocessing, oversampling, and XGBoost classifier
xgb_oversampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('sampler', oversampler),
    ('classifier', XGBClassifier(
        n_estimators=100,   # Boosting rounds
        learning_rate=0.1,  # Standard learning rate
        max_depth=5,        # Moderate depth to balance learning capacity
        subsample=0.8,      # Randomly sample 80% of data per tree
        min_child_weight=1,   # default; allows splitting with minimal constraints
        colsample_bytree=0.8,  # Use 80% of features for each tree
        objective='binary:logistic',
        eval_metric='logloss',  # Standard for binary classification
        gamma=0,                # No split regularization initially
        #scale_pos_weight=0.5,   # Adjust for 66%/33% class imbalance
        use_label_encoder=False,  # Prevents warnings
        random_state=42,
    ))
])

# Fit the pipeline on training data
xgb_oversampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = xgb_oversampled_pipeline.predict(X_test)


xgb_oversampled_result = get_classification_metrics(y_test, y_pred)
xgb_oversampled_result


results['xgb_up'] = xgb_oversampled_result


# Create pipeline with preprocessing, undersampling, and XGBoost classifier
xgb_undersampled_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('undersampler', undersampler),
    ('classifier', XGBClassifier(
        n_estimators=100,   # Boosting rounds
        learning_rate=0.1,  # Standard learning rate
        max_depth=5,        # Moderate depth to balance learning capacity
        subsample=0.8,      # Randomly sample 80% of data per tree
        min_child_weight=1,   # default; allows splitting with minimal constraints
        colsample_bytree=0.8,  # Use 80% of features for each tree
        objective='binary:logistic',
        eval_metric='logloss',  # Standard for binary classification
        gamma=0,                # No split regularization initially
        #scale_pos_weight=0.5,   # Adjust for 66%/33% class imbalance
        use_label_encoder=False,  # Prevents warnings
        random_state=42,
    ))
])

# Fit the pipeline on training data
xgb_undersampled_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = xgb_undersampled_pipeline.predict(X_test)


xgb_undersampled_result = get_classification_metrics(y_test, y_pred)
xgb_undersampled_result


results['xgb_down'] = xgb_undersampled_result


# Create stratified k-fold cross-validation
stratified_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Base pipeline with oversampling
xgb_base_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('classifier', XGBClassifier(
        objective='binary:logistic',
        random_state=42,
        use_label_encoder=False,
        eval_metric='logloss',
        verbosity=0 # to suppress warnings
    ))
])


# Phase 1: Tune tree-based parameters
param_grid1 = {
    'classifier__max_depth': [3, 4, 5, 6, 7],
    'classifier__min_child_weight': [1, 3, 5, 7],
    'classifier__gamma': [0, 0.1, 0.2, 0.3]
}

grid_search1 = GridSearchCV(
    xgb_base_pipeline,
    param_grid1,
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1
)

grid_search1.fit(X_train, y_train)
best_params1 = grid_search1.best_params_
print(f"Phase 1 best parameters: {best_params1}")

Fitting 5 folds for each of 80 candidates, totalling 400 fits
Phase 1 best parameters: {'classifier__gamma': 0.3, 'classifier__max_depth': 3, 'classifier__min_child_weight': 7}


# Phase 2: Tune learning rate, n_estimators, subsample, and colsample_bytree with Random Search
# Also use learnings gathered from Phase 1

xgb_phase2_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('classifier', XGBClassifier(
        objective='binary:logistic',
        random_state=42,
        use_label_encoder=False,
        eval_metric='logloss',
        verbosity=0, # to suppress warnings
        # learnings from phase 1 incorporated
        **{k.replace('classifier__', ''): v for k, v in best_params1.items()}
    ))
])

param_distributions2 = {
    'classifier__learning_rate': uniform(0.01, 0.29),  # 0.01 to 0.3
    'classifier__n_estimators': randint(50, 300),
    'classifier__subsample': uniform(0.7, 0.3),  # 0.7 to 1.0
    'classifier__colsample_bytree': uniform(0.7, 0.3)  # 0.7 to 1.0
}
random_search2 = RandomizedSearchCV(
    xgb_phase2_pipeline,
    param_distributions=param_distributions2,
    n_iter=50,
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1,
    random_state=42
)

random_search2.fit(X_train, y_train)
best_params2 = random_search2.best_params_
print(f"Phase 2 best parameters: {best_params2}")

Fitting 5 folds for each of 50 candidates, totalling 250 fits
Phase 2 best parameters: {'classifier__colsample_bytree': 0.7021198915659151, 'classifier__learning_rate': 0.01668810326201057, 'classifier__n_estimators': 268, 'classifier__subsample': 0.8835558684167137}


# Phase 3: Fine-tune regularization parameters with Grid Search

# Combine best parameters from Phase 1 and 2
best_params_combined = {**{k.replace('classifier__', ''): v for k, v in best_params1.items()},
                        **{k.replace('classifier__', ''): v for k, v in best_params2.items()}}

xgb_phase3_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    ('classifier', XGBClassifier(
        objective='binary:logistic',
        random_state=42,
        use_label_encoder=False,
        eval_metric='logloss',
        verbosity=0, # to suppress warnings
        **best_params_combined # incorporate previous findings/learnings
    ))
])

param_grid3 = {
    'classifier__reg_alpha': [0, 0.1, 0.5, 1.0],
    'classifier__reg_lambda': [0.1, 0.5, 1.0, 5.0]
}

grid_search3 = GridSearchCV(
    xgb_phase3_pipeline,
    param_grid3,
    cv=stratified_cv,
    scoring='f1',
    n_jobs=-1,
    verbose=1
)

grid_search3.fit(X_train, y_train)
best_params3 = grid_search3.best_params_
print(f"Phase 3 best parameters: {best_params3}")

Fitting 5 folds for each of 16 candidates, totalling 80 fits
Phase 3 best parameters: {'classifier__reg_alpha': 0, 'classifier__reg_lambda': 1.0}


# Final model with all best parameters
all_best_params = {**best_params_combined,
                  **{k.replace('classifier__', ''): v for k, v in best_params3.items()}}

xgb_final_tuned_pipeline = ImbPipeline([
    ('preprocessor', preprocessor),
    # uncomment this if needed (as the performance with & without was almost same, So commented for NoW)
    #('oversampler', oversampler),
    ('classifier', XGBClassifier(
        objective='binary:logistic',
        random_state=42,
        use_label_encoder=False,
        eval_metric='logloss',
        **all_best_params
    ))
])

# Fit final tuned model
xgb_final_tuned_pipeline.fit(X_train, y_train)

# Make predictions
y_pred = xgb_final_tuned_pipeline.predict(X_test)


xgb_final_tuned_result = get_classification_metrics(y_test, y_pred)
xgb_final_tuned_result


results['xgb_tuned'] = xgb_final_tuned_result


# Feature Importance
model = xgb_final_tuned_pipeline.named_steps['classifier']
feat_importances = get_feature_importances(model, transformed_columns)
feat_importances


plot_feature_importance(feat_importances, top_n=10)


def compare_model_performance(model_metrics, model_names=None):
    """
    Compare performance metrics of different decision tree models

    Parameters:
    -----------
    model_metrics : list of DataFrames
        Performance metric DataFrames for each model
    model_names : list of str, optional
        Names to use for each model. If not provided, will use generic names

    Returns:
    --------
    DataFrame
        Combined performance metrics with models as columns
    """
    # Concatenate all model metrics
    comparison_df = pd.concat([m.T for m in model_metrics], axis=1)

    # Set column names
    if model_names:
        comparison_df.columns = model_names
    else:
        comparison_df.columns = [f"Model {i+1}" for i in range(len(model_metrics))]

    return comparison_df


model_names = []
model_metrics = []

for model_id, model_result in results.items():
    model_names.append(model_id)
    model_metrics.append(model_result)

comparision_result_df = compare_model_performance(model_metrics, model_names)
comparision_result_df.T

case_status	Certified	Denied
no_of_employees
Very Small (10-99)	62.66	37.34
Small (100-999)	66.20	33.80
Medium (1K-10K)	66.99	33.01
Large (10K-100K)	68.07	31.93
Very Large (>100K)	70.11	29.89

case_status	Certified	Denied
yr_of_estab
Historic (1800-1950)	66.19	33.81
Post-war (1951-1975)	66.62	33.38
Modern (1976-1995)	65.44	34.56
Recent (1996-2005)	66.04	33.96
New (2006-2016)	69.63	30.37

case_status	Certified	Denied
region_of_employment
Island	22600	14900
Midwest	325300	105400
Northeast	452600	266900
South	491300	210400
West	410000	248600

case_status	Certified	Denied
region_of_employment
Island	60.27	39.73
Midwest	75.53	24.47
Northeast	62.90	37.10
South	70.02	29.98
West	62.25	37.75

	Accuracy	Recall	Precision	F1
dt	0.66	0.75	0.74	0.74
dt_up	0.64	0.70	0.75	0.72
dt_down	0.63	0.62	0.77	0.69
dt_tuned	0.73	0.91	0.74	0.82
bc	0.71	0.92	0.72	0.81
bc_up	0.71	0.79	0.77	0.78
bc_down	0.68	0.69	0.80	0.74
bc_tuned	0.73	0.90	0.75	0.82
rf	0.73	0.85	0.77	0.81
rf_up	0.70	0.76	0.78	0.77
rf_down	0.68	0.67	0.81	0.74
rf_tuned	0.74	0.87	0.77	0.82
ada	0.71	0.83	0.76	0.79
ada_up	0.71	0.80	0.77	0.79
ada_down	0.67	0.67	0.80	0.73
ada_tuned	0.74	0.88	0.76	0.82
gb	0.74	0.87	0.77	0.82
gb_up	0.71	0.74	0.81	0.78
gb_down	0.70	0.71	0.82	0.76
gb_tuned	0.74	0.87	0.77	0.82
xgb	0.71	0.73	0.81	0.77
xgb_up	0.72	0.77	0.80	0.78
xgb_down	0.70	0.71	0.82	0.76
xgb_tuned	0.74	0.88	0.77	0.82

	case_id	continent	education_of_employee	has_job_experience	requires_job_training	no_of_employees	yr_of_estab	region_of_employment	prevailing_wage	unit_of_wage	full_time_position	case_status
0	EZYV01	Asia	High School	N	N	14513	2007	West	592.20	Hour	Y	Denied
1	EZYV02	Asia	Master's	Y	N	2412	2002	Northeast	83425.65	Year	Y	Certified
2	EZYV03	Asia	Bachelor's	N	Y	44444	2008	West	122996.86	Year	Y	Denied
3	EZYV04	Asia	Bachelor's	N	N	98	1897	West	83434.03	Year	Y	Denied
4	EZYV05	Africa	Master's	Y	N	1082	2005	South	149907.39	Year	Y	Certified

	count	mean	std	min	25%	50%	75%	max
no_of_employees	25480.00	5667.04	22877.93	-26.00	1022.00	2109.00	3504.00	602069.00
yr_of_estab	25480.00	1979.41	42.37	1800.00	1976.00	1997.00	2005.00	2016.00
prevailing_wage	25480.00	74455.81	52815.94	2.14	34015.48	70308.21	107735.51	319210.27

case_status	Certified	Denied
yr_of_estab
Legacy (Before 1950)	66.19	33.81
Traditional Era (1950-1979)	66.50	33.50
Growth Phase (1980-1999)	65.51	34.49
New Age (2000-2016)	68.26	31.74

case_status	Certified	Denied
prevailing_wage
Entry-level ($0-40K)	60.82	39.18
Mid-level ($40K-80K)	69.38	30.62
Professional ($80K-150K)	68.86	31.14
Executive (>$150K)	70.08	29.92

case_status	Certified	Denied
education_of_employee
Bachelor's	636700	386700
Doctorate	191200	28000
High School	116400	225600
Master's	757500	205900

case_status	Certified	Denied
unit_of_wage
Hour	74700	141000
Month	5500	3400
Week	16900	10300
Year	1604700	691500

case_status	Certified	Denied
unit_of_wage
Hour	34.63	65.37
Month	61.80	38.20
Week	62.13	37.87
Year	69.89	30.11

	no_of_employees	company_size
0	14513	Enterprise (>10000)
1	2412	Large (1001-10000)
2	44444	Enterprise (>10000)
3	98	Small (51-250)
4	1082	Large (1001-10000)

	yr_of_estab	company_era
0	2007	Digital Transition Era (2000-2010)
1	2002	Digital Transition Era (2000-2010)
2	2008	Digital Transition Era (2000-2010)
3	1897	Legacy (Before 1950)
4	2005	Digital Transition Era (2000-2010)

	case_id	continent	education_of_employee	has_job_experience	requires_job_training	no_of_employees	yr_of_estab	region_of_employment	prevailing_wage	unit_of_wage	full_time_position	case_status	company_size	company_era
599	EZYV600	Asia	Doctorate	Y	N	2467	1800	South	62487.14	Year	Y	Certified	Large (1001-10000)	Legacy (Before 1950)
1279	EZYV1280	Asia	Bachelor's	N	Y	2014	1800	South	68613.70	Year	Y	Certified	Large (1001-10000)	Legacy (Before 1950)
1472	EZYV1473	Europe	Doctorate	Y	N	1561	1800	South	43420.11	Year	Y	Certified	Large (1001-10000)	Legacy (Before 1950)
1882	EZYV1883	South America	Master's	N	Y	3814	1800	Northeast	78340.74	Year	Y	Denied	Large (1001-10000)	Legacy (Before 1950)
6083	EZYV6084	Asia	Master's	N	N	1604	1800	Midwest	73305.46	Year	Y	Certified	Large (1001-10000)	Legacy (Before 1950)

case_status	Certified	Denied
full_time_position
N	68.53	31.47
Y	66.58	33.42

	education_of_employee	company_size	company_era	wage_category	has_job_experience	full_time_position	continent_Asia	continent_North America	region_of_employment_Midwest	region_of_employment_Northeast	region_of_employment_South	region_of_employment_West	unit_of_wage_Year	no_of_employees	yr_of_estab	prevailing_wage
3516	1.00	3.00	2.00	1.00	1.00	1.00	1.00	0.00	0.00	0.00	1.00	0.00	1.00	4069.00	1984.00	61313.77
13759	1.00	3.00	1.00	0.00	1.00	1.00	1.00	0.00	1.00	0.00	0.00	0.00	1.00	3022.00	1961.00	36588.11
6019	1.00	2.00	1.00	3.00	0.00	1.00	0.00	1.00	0.00	0.00	1.00	0.00	1.00	751.00	1971.00	210280.88
779	2.00	3.00	1.00	1.00	1.00	1.00	1.00	0.00	0.00	1.00	0.00	0.00	1.00	1113.00	1968.00	46457.53
22792	1.00	2.00	1.00	2.00	1.00	1.00	0.00	1.00	0.00	0.00	0.00	1.00	1.00	796.00	1979.00	89542.27

	feature	importance
0	education_of_employee	0.46
4	has_job_experience	0.21
18	unit_of_wage_Year	0.11
8	continent_Europe	0.07
21	prevailing_wage	0.07
12	region_of_employment_Midwest	0.05
9	continent_North America	0.01
11	continent_South America	0.01
6	full_time_position	0.01
20	yr_of_estab	0.00
19	no_of_employees	0.00
7	continent_Asia	0.00
13	region_of_employment_Northeast	0.00
1	company_size	0.00
14	region_of_employment_South	0.00
15	region_of_employment_West	0.00
16	unit_of_wage_Month	0.00
17	unit_of_wage_Week	0.00
2	company_era	0.00
10	continent_Oceania	0.00
3	wage_category	0.00
5	requires_job_training	0.00

	feature	importance
0	education_of_employee	0.29
21	prevailing_wage	0.16
4	has_job_experience	0.11
19	no_of_employees	0.10
20	yr_of_estab	0.08
18	unit_of_wage_Year	0.04
8	continent_Europe	0.03
2	company_era	0.02
3	wage_category	0.02
12	region_of_employment_Midwest	0.02
6	full_time_position	0.02
15	region_of_employment_West	0.02
14	region_of_employment_South	0.02
7	continent_Asia	0.01
13	region_of_employment_Northeast	0.01
1	company_size	0.01
5	requires_job_training	0.01
9	continent_North America	0.01
11	continent_South America	0.00
17	unit_of_wage_Week	0.00
10	continent_Oceania	0.00
16	unit_of_wage_Month	0.00

	feature	importance
0	education_of_employee	0.25
19	no_of_employees	0.20
4	has_job_experience	0.16
21	prevailing_wage	0.11
20	yr_of_estab	0.05
12	region_of_employment_Midwest	0.04
18	unit_of_wage_Year	0.04
6	full_time_position	0.03
13	region_of_employment_Northeast	0.03
8	continent_Europe	0.02
15	region_of_employment_West	0.02
5	requires_job_training	0.01
9	continent_North America	0.01
7	continent_Asia	0.01
11	continent_South America	0.01
14	region_of_employment_South	0.00
2	company_era	0.00
10	continent_Oceania	0.00
16	unit_of_wage_Month	0.00
17	unit_of_wage_Week	0.00
1	company_size	0.00
3	wage_category	0.00

	feature	importance
0	education_of_employee	0.45
4	has_job_experience	0.16
21	prevailing_wage	0.10
18	unit_of_wage_Year	0.07
8	continent_Europe	0.05
12	region_of_employment_Midwest	0.03
19	no_of_employees	0.02
15	region_of_employment_West	0.02
9	continent_North America	0.02
20	yr_of_estab	0.01
14	region_of_employment_South	0.01
7	continent_Asia	0.01
13	region_of_employment_Northeast	0.01
11	continent_South America	0.01
3	wage_category	0.01
5	requires_job_training	0.01
6	full_time_position	0.01
2	company_era	0.00
1	company_size	0.00
17	unit_of_wage_Week	0.00
10	continent_Oceania	0.00
16	unit_of_wage_Month	0.00

	feature	importance
0	education_of_employee	0.24
4	has_job_experience	0.18
21	prevailing_wage	0.09
18	unit_of_wage_Year	0.09
8	continent_Europe	0.08
12	region_of_employment_Midwest	0.05
14	region_of_employment_South	0.05
15	region_of_employment_West	0.04
13	region_of_employment_Northeast	0.03
9	continent_North America	0.03
6	full_time_position	0.03
3	wage_category	0.02
11	continent_South America	0.02
7	continent_Asia	0.02
17	unit_of_wage_Week	0.01
5	requires_job_training	0.01
19	no_of_employees	0.01
20	yr_of_estab	0.01
2	company_era	0.01
1	company_size	0.00
10	continent_Oceania	0.00
16	unit_of_wage_Month	0.00

Easy Visa¶

Problem Statement¶

Data background and Contents¶

Preparation¶

Univariate Analysis (EDA)¶

1. Continent¶

2. Education of Employee¶

3. Has Job Experience¶

4. Requires Job Training¶

5. Number of Employees¶

6. Year of Establishment¶

7 Region of Employment¶

8. Prevailing Wage (Wage Paid Similalry)¶

9. Unit of Wage¶

10. Full Time Position¶

11. Case Status¶

MultiVariate Analysis¶

Helper (Python)¶

1. Continent vs Case Status¶

2. Has Job Experience vs Case Status¶

3. Education vs Case Status¶

4. Education of Employee vs Has Job Experience¶

5. Year of Establishment vs Case Status¶

6 Year of Establishment & Continent¶

7. Number of Employees vs Case Status¶

8. Region of Employment vs Case Status¶

9. Region and Company Age¶

10 Prevailing Wage vs Case Status¶

11. Prevailing Wage and Region of Employment¶

11. Unit of Wage vs Case Status¶

12. Full time Position vs Case Status¶

NOTES - Tree Models¶

PreProcessing¶

1. Negative Value Treatment¶

2. Outlier Detection¶

Binning¶

📌 REMEMBER¶

Drop unneeded columns¶

Feature Engineering¶

Ordinal Encoding Prep¶

One Hot Encoding Prep¶

Binary Encoding Prep¶

Train Test Split¶

Modeling¶

Utils¶

GLOBALS¶

PreProcessor¶

UpSampler¶

UnderSampling¶

PipeLines | Modeling¶

1. Default Decision Tree¶

2. Decision Tree + UpSampling¶

3. Decision Tree + DownSampling¶

4. Decision Tree (Tuned)¶

5 Bagging Classifier¶

6 Bagging Classifier + OverSampling¶

7. Bagging Classifier + UnderSampling¶

8. Bagging Classifier (Tuning)¶

9. Random Forest¶

10 Random Forest + Oversampling¶

11 Random Forest + Undersampling¶

12 Random Forest (Tuning)¶

13 AdaBoost¶

14 Adaboost + OverSampling¶

15 AdaBoost + DownSampling¶

16 AdaBoost (Tuning)¶

17. Gradient Boost¶

18 Gradient Boosting + OverSampling¶

19 Gradient Boosting + UnderSampling¶

20 Gradient Boosting (Tuning)¶

21 XgBoosting¶

22 XgBoost + OverSampling¶

23 XgBoosting + UnderSampling¶

24 XgBoosting + Tuning¶

Model Performance Comparision¶

Actionable Insights and Recommendations¶

Key Takeaways¶

Recommendations For Job Seekers¶

Recommendations For Job Employers¶

Recommendations for Visa-Approving Agencies¶